# (c) cavaliba.com - home - migrator.py # start / bootstrap / update import logging import os import sys import time import uuid import yaml from django.conf import settings from django.utils import timezone import app_home.cache as cache from app_data.loader import load_broker from app_data.models import DataSchema from app_data.permissions import bootstrap_permissions, permission_all_keynames from app_data.registry import registry_get_key, registry_set_key from app_home.configuration import sync_configuration logger = logging.getLogger(__name__) def cavaliba_start(): """ init DB or apply migration when cavaliba starts called from management command cavaliba start (from start_dev.sh or docker-entrypoint.sh) Creates Registry Entries - DB_VERSION - LAST_START - INSTANCE_UUID """ logger.info("cavaliba_start() ...") instance_uuid = registry_get_key(key="INSTANCE_UUID") logger.info(f"instance UUID: {instance_uuid}") db_version = registry_get_key(key="DB_VERSION") logger.info(f"db version: {db_version}") logger.info(f"code version: {settings.CAVALIBA_VERSION}") # bootstrap once if db_version is None or db_version == 0: db_version = cavaliba_bootstrap() logger.info(f"bootstrap done to {db_version}") # configuration always sync_configuration() logger.info("configuration synced") # migrate / update always db_version = cavaliba_db_update(db_version) # seed builtin apikey secret, once sync_apikey_secret() # force admin account password from env, every start sync_admin_password() # final check if settings.CAVALIBA_VERSION == db_version: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START", value=timestamp) logger.info(f"start done at {timestamp}") return else: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START_FAILED", value=timestamp) logger.critical( f"start FAILED - code version {settings.CAVALIBA_VERSION} != db version {db_version}" ) def cavaliba_bootstrap(): """init an empty database with builtin/init yaml files""" initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") logger.info(f"first start, init/bootstraping new DB : {initdir}") # load permissions first perm_file = os.path.join(initdir, "00_permission.yml") logger.info(f"bootstrap - {perm_file}") with open(perm_file) as f: content = yaml.load(f, Loader=yaml.SafeLoader) bootstrap_permissions(content) # load all other yml files in lexicographic order aaa = {"perms": permission_all_keynames()} other_files = sorted( item.path for item in os.scandir(initdir) if item.is_file(follow_symlinks=False) and item.path.endswith(".yml") and os.path.basename(item.path) != "00_permission.yml" ) for filename in other_files: logger.info(f"bootstrap - {filename}") with open(filename) as f: try: content = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f"bootstrap - SKIP - YAML error in {filename}: {e}") continue if content: load_broker(datalist=content, aaa=aaa) db_version = "4.2.0" registry_set_key(key="DB_VERSION", value=db_version) new_uuid = str(uuid.uuid4()) registry_set_key(key="INSTANCE_UUID", value=new_uuid) return db_version def cavaliba_db_update(db_version): """apply various migration/update until db_version reaches code version""" if db_version == settings.CAVALIBA_VERSION: logger.info("no DB migration needed") # migration to 4.0.0 - no longer supported, refuse to start if db_version < "4.0.0": logger.critical("=" * 70) logger.critical(f"STARTUP REFUSED - DB version is {db_version}") logger.critical("Direct upgrade from v3 is no longer supported by this build.") logger.critical("Upgrade to a 4.0.X or 4.1.X release first, then to this version.") logger.critical("=" * 70) time.sleep(10) sys.exit(1) # migration to 4.2.0 if db_version < "4.2.0": from app_user.models import SirenePermission logger.info("update to 4.2.0") # p_keyname_rename message = "add p_keyname_rename builtin permission" _nouse = reload_builtin( message, "00_permission.yml", ["_permission:p_keyname_rename"], "init" ) # code_editor sidebar entry message = "add Code Editor builtin sidebar entry" _nouse = reload_builtin(message, "090_dashboard.yml", ["_home:code_editor"], "init") # revision log sidebar entry message = "add Revision Log builtin sidebar entry" _nouse = reload_builtin(message, "090_dashboard.yml", ["_home:revision"], "init") # new ipam_ip schema message = "add ipam_ip builtin schema" _nouse = reload_builtin(message, "220-ipam.yml", ["_schema:ipam_ip"], "init") # new user default dataview message = "add user_default builtin dataview" _nouse = reload_builtin(message, "010_user.yml", ["_dataview:user_default"], "init") # remove permissions marked "_action: delete" in 00_permission.yml # direct ORM delete: permission_delete() refuses builtin permissions SirenePermission.objects.filter(keyname="p_schema_create").delete() SirenePermission.objects.filter(keyname="p_schema_update").delete() SirenePermission.objects.filter(keyname="p_schema_delete").delete() SirenePermission.objects.filter(keyname="p_schemafield_read").delete() SirenePermission.objects.filter(keyname="p_schemafield_create").delete() SirenePermission.objects.filter(keyname="p_schemafield_update").delete() SirenePermission.objects.filter(keyname="p_schemafield_delete").delete() logger.info(" OK - removed obsolete schema/schemafield permissions") db_version = "4.2.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.4.0 if db_version < "4.4.0": logger.info("update to 4.4.0") # new bulk_delete_disabled builtin pipeline message = "add bulk_delete_disabled builtin pipeline" _nouse = reload_builtin( message, "040_pipeline.yml", ["_pipeline:bulk_delete_disabled"], "init" ) # p_api_load - gates POST /api/load/ message = "add p_api_load builtin permission" _nouse = reload_builtin(message, "00_permission.yml", ["_permission:p_api_load"], "init") # p_api_rawfile - gates POST /api/rawfile/ (renamed from /api/import/, # which used p_data_import - a UI-shared permission left untouched; # existing API keys need p_api_rawfile granted to keep using this endpoint) message = "add p_api_rawfile builtin permission" _nouse = reload_builtin(message, "00_permission.yml", ["_permission:p_api_rawfile"], "init") db_version = "4.4.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.5.0 if db_version < "4.5.0": logger.info("update to 4.5.0") # p_api_conf - gates /api/conf/ (GET also needs p_conf_view/p_conf_admin, # POST also needs p_conf_admin - see app_data/api/conf.py) message = "add p_api_conf builtin permission" _nouse = reload_builtin(message, "00_permission.yml", ["_permission:p_api_conf"], "init") db_version = "4.5.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.6.0 # dev trick: while CAVALIBA_VERSION is tagged "4.6.0-RC..." this block re-runs # every start regardless of DB_VERSION, so there's no need to reset the # DB_VERSION registry key by hand while iterating on it. Tied to "4.6.0" # specifically so it stops applying the moment work moves to the next # version's block (e.g. once CAVALIBA_VERSION becomes "4.7.0-RC..."). # Caveat: every step below runs on each repeated pass while "-RC" is present - # only safe for idempotent steps (reload_builtin, load/save round-trips). # A non-idempotent step (e.g. unconditionally appending to a list field) # must guard itself against running twice. if db_version < "4.6.0" or settings.CAVALIBA_VERSION.startswith("4.6.0-RC"): logger.info("update to 4.6.0") # csv_delimiter/encoding/keyfield/classname were removed from Pipeline - # they only ever lived as ad-hoc keys inside a _pipeline instance's # "content" YAML. Then the "tasks:" wrapper key itself was dropped - # "content" is now directly the rule list. Round-trip every _pipeline # instance's raw content and write back a clean flat-list content. # Parsed by hand (not via Pipeline.from_instance, which assumes the # current/new shape) because a dev DB bootstrapped under an earlier # 4.6.0-RC pass may still have the intermediate {tasks: [...]} shape # on disk - both shapes are accepted here so this step stays # idempotent across repeated -RC runs. from app_data.data import Instance message = "clean deprecated fields from _pipeline content" aaa = {"perms": permission_all_keynames()} updated = 0 for instance in Instance.stream_classname(classname="_pipeline"): old_raw = instance.get_attribute_first("content") or "" try: parsed = yaml.safe_load(old_raw) except yaml.YAMLError: parsed = None if isinstance(parsed, dict): rules = parsed.get("tasks", []) elif isinstance(parsed, list): rules = parsed else: rules = [] new_raw = yaml.dump(rules, allow_unicode=True, sort_keys=False) if new_raw.strip() != old_raw.strip(): instance.set_field_value_single(fieldname="content", value=new_raw) instance.save(aaa=aaa, action="update", skip_revision=True) updated += 1 logger.info(f" OK - {message} ({updated} instance(s))") # texteditor view was renamed to code_editor (URL path # /data/private/texteditor/ -> /data/private/code_editor/) - refresh the # builtin _home:code_editor dashboard/sidebar entry's url field message = "update _home:code_editor url to /data/private/code_editor/" _nouse = reload_builtin( message, "090_dashboard.yml", ["_home:code_editor"], force_action="update" ) # p_code_editor - gates the RAW YAML/JSON Code Editor, split out from # p_data_import (which now only gates the CSV/YAML/JSON Import Tool) message = "add p_code_editor builtin permission" _nouse = reload_builtin(message, "00_permission.yml", ["_permission:p_code_editor"], "init") # new builtin api_builtin apikey (secret intentionally empty here - seeded # separately from CAVALIBA_APIKEY_SECRET env var by sync_apikey_secret(), # if set) message = "add api_builtin builtin apikey" _nouse = reload_builtin(message, "005_apikey.yml", ["_apikey:api_builtin"], "init") db_version = "4.6.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.8.0 # dev trick: while CAVALIBA_VERSION is tagged "4.8.0-RC..." this block re-runs # every start regardless of DB_VERSION - see the 4.6.0 block above for the # full explanation. Steps 1/2 (reload_builtin) are naturally idempotent; # step 3 (instance migration) guards its own idempotency - see below. if db_version < "4.8.0" or settings.CAVALIBA_VERSION.startswith("4.8.0-RC"): logger.info("update to 4.8.0") # 1 - ipam_ip schema reworked: keyname_mode switched to "hexip" (keyname # derived from displayname, an IPv4 address) and a new "collected" # field added (true for IPs discovered automatically from ipv4 fields # on other schemas, as opposed to manually created ones) message = "reload builtin ipam_ip schema (hexip keyname_mode, new collected field)" _nouse = reload_builtin(message, "220-ipam.yml", ["_schema:ipam_ip"], "update") # 2 - ipam_ip_default dataview: add displayname/collected columns message = "reload builtin ipam_ip_default dataview" _nouse = reload_builtin(message, "220-ipam.yml", ["_dataview:ipam_ip_default"], "update") # 3 - migrate existing ipam_ip instances predating keyname_mode=hexip # (must run after step 1, so the schema is already in hexip mode) migrate_ipam_ip_hexip() # 4 - IPAM search/menu landing page moved from app_ipam (/ipam/private/) # to app_data (/data/private/ipam/) - refresh the builtin _home:ipam # dashboard/sidebar entry's url field to match message = "update _home:ipam url to /data/private/ipam/" _nouse = reload_builtin(message, "220-ipam.yml", ["_home:ipam"], "update") db_version = "4.8.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.9.0 # dev trick: while CAVALIBA_VERSION is tagged "4.9.0-RC..." this block re-runs # every start regardless of DB_VERSION - see the 4.6.0 block above for the # full explanation. Steps 1/4 (reload_builtin) are naturally idempotent; # steps 2/3 guard their own idempotency - see migrate_ipam_subnet_keyname() # and the DataSchema delete below (a no-op once the row is already gone). if db_version < "4.9.0" or settings.CAVALIBA_VERSION.startswith("4.9.0-RC"): logger.info("update to 4.9.0") # 1 - ipam_subnet schema: pick up keyname_mode="dataformat ipv4 subnet # strict" + keyname_help/displayname_help/labels. The deprecated # "subnet" field is simply absent from the reloaded content - omitting # a field from reload_builtin()'s payload does not delete it on its # own, that's handled explicitly in step 3 below. message = "reload builtin ipam_subnet schema (dataformat keyname_mode)" _nouse = reload_builtin(message, "220-ipam.yml", ["_schema:ipam_subnet"], "update") # 2 - re-key any ipam_subnet instance predating this schema change # whose keyname doesn't already match its (about to be deleted) # "subnet" field value - must run before step 3 deletes that field. # Uses the keyname.py rename() cascade (not a plain re-save) since # this keyname_mode never auto-derives keyname the way ipam_ip's # hexip mode does - and rename() is what fixes up any ipam_vlan.subnet # field still pointing at the old keyname. migrate_ipam_subnet_keyname() # 3 - drop the now-deprecated "subnet" field definition itself # (direct ORM delete - reload_builtin() can't express a field removal # once the field is gone from 220-ipam.yml's own content, matching # the "Delete a builtin permission" pattern in .claude/CLAUDE.md) DataSchema.objects.filter(classname="ipam_subnet", keyname="subnet").delete() cache.cache2_schema.delete("ipam_subnet") # 4 - ipam_subnet dataview: drop the now-redundant "subnet" column message = "reload builtin ipam_subnet dataview (drop subnet column)" _nouse = reload_builtin(message, "220-ipam.yml", ["_dataview:ipam_subnet"], "update") db_version = "4.9.0" registry_set_key(key="DB_VERSION", value=db_version) # HERE NEW MIGRATIONS FOR NEXT VERSIONS # always end to code version db_version = settings.CAVALIBA_VERSION registry_set_key(key="DB_VERSION", value=db_version) db_version_date = settings.CAVALIBA_VERSION_DATE registry_set_key(key="DB_VERSION_DATE", value=db_version_date) return db_version def sync_apikey_secret(): """set api_builtin apikey secret from CAVALIBA_APIKEY_SECRET, only if currently empty""" secret_env = settings.CAVALIBA_APIKEY_SECRET if not secret_env: return from app_data import crypto from app_data.data import Instance instance = Instance.from_keyname(classname="_apikey", keyname="api_builtin") if not instance: logger.warning("sync_apikey_secret: api_builtin apikey not found, skipping") return if instance.get_attribute_first("secret"): logger.info("sync_apikey_secret: secret already set, skipping") return hashed = crypto.hash_create(secret_env) instance.set_field_value_single(fieldname="secret", value=hashed) aaa = {"perms": permission_all_keynames()} instance.save(aaa=aaa, action="update", skip_revision=True) logger.info("sync_apikey_secret: secret set from CAVALIBA_APIKEY_SECRET env var") def sync_admin_password(): """force the Django "admin" superuser password from CAVALIBA_ADMIN_PASSWORD env var, every start. If the env var is not set, lock the account (is_active=False) instead.""" from django.contrib.auth import get_user_model UserModel = get_user_model() try: admin = UserModel.objects.get(username="admin") except UserModel.DoesNotExist: logger.warning("sync_admin_password: admin account not found, skipping") return password_env = settings.CAVALIBA_ADMIN_PASSWORD if not password_env: if admin.is_active: admin.is_active = False admin.save() logger.warning( "sync_admin_password: CAVALIBA_ADMIN_PASSWORD not set, admin account locked" ) return changed = False if not admin.check_password(password_env): admin.set_password(password_env) changed = True if not admin.is_active: admin.is_active = True changed = True if changed: admin.save() logger.info( "sync_admin_password: admin password synced from CAVALIBA_ADMIN_PASSWORD env var" ) # ---- def reload_builtin(message="", filename=None, objects=None, force_action="create"): """ reload builtin/init/040_pipeline.yml for new builtin bulk pipelines objects: [ "schema:keyname", "schema:keyname", ... ] or ['*'] """ initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") pipeline_file = os.path.join(initdir, filename) aaa = {"perms": permission_all_keynames()} try: with open(pipeline_file) as f: content_yml = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f" KO - {message} : {e}") return False if not content_yml: logger.error(f" KO - {message} : no content") return False if not objects: logger.error(f" KO - {message} : no object provided") return False # filter objects to load if objects == ["*"]: datalist = content_yml else: # objects is a list of "classname:keyname" strings wanted = set() for entry in objects: parts = entry.split(":", 1) if len(parts) == 2: wanted.add((parts[0], parts[1])) datalist = [ item for item in content_yml if (item.get("classname"), item.get("keyname")) in wanted ] if not datalist: logger.error(f" KO - {message} : no matching objects to {objects}") return False try: load_broker(datalist=datalist, aaa=aaa, force_action=force_action) except Exception as e: logger.error(f" KO - {message} : {e}") return False logger.info(f" OK - {message}") # ---- def migrate_ipam_ip_hexip(): """ v4.8.0: ipam_ip instances predating keyname_mode=hexip still have their dotted-IPv4 address as keyname directly (e.g. "10.1.2.3"). For each one: copy that keyname into displayname, leave "collected" alone (didn't exist before this version, so it's already unset -> reads as False), then re-save so hexip (the ipam_ip schema must already be in hexip mode - see the reload_builtin() call for "_schema:ipam_ip" run before this) derives the new hex keyname on the same row. save()'s normal EAV rewrite only knows the *new* keyname (see app_data/eav.py EavBatch.save_bulk()) - unlike app_data/keyname.py rename(), it never cleans up the EAV row still sitting under the old dotted-IPv4 keyname, so that row is purged explicitly here. Skips any instance whose keyname is no longer a plain dotted IPv4 - keeps this idempotent across repeated calls (e.g. the 4.8.0 migration block re-running every start while CAVALIBA_VERSION is tagged "-RC"). """ from app_data.data import Instance from app_data.fieldtypes.field_ipv4 import check_valid_ipv4 from app_data.models import DataEAV message = "migrate ipam_ip instances to hexip keyname" aaa = {"perms": permission_all_keynames()} scanned = 0 migrated = 0 for instance in Instance.stream_classname(classname="ipam_ip"): scanned += 1 old_keyname = instance.keyname if not check_valid_ipv4(old_keyname): continue instance.displayname = old_keyname instance.save(aaa=aaa, action="update", skip_revision=True) if instance.keyname != old_keyname: DataEAV.objects.filter( classname="ipam_ip", keyname=old_keyname, iid=instance.id ).delete() migrated += 1 if migrated % 100 == 0: logger.info(f" ... {message}: {migrated} done ({scanned} scanned)") logger.info(f" OK - {message} ({migrated} migrated, {scanned} scanned)") # ---- def migrate_ipam_subnet_keyname(): """ v4.9.0: ipam_subnet instances predating keyname_mode="dataformat ipv4 subnet strict" may have a freeform keyname unrelated to their "subnet" field's actual CIDR value (that field is being deleted in this same migration - see cavaliba_db_update()). For each mismatched instance, rename() it to the subnet field's value - this is a full cascading rename (not a raw keyname edit), so any ipam_vlan.subnet field still pointing at the old keyname gets fixed up too. Skips: instances already matching (keyname == subnet value - true for all 4 builtin seed rows, so this is a no-op on a fresh 4.8.0 install), instances with no subnet value set, and instances whose subnet value isn't itself a valid strict/aligned CIDR (logged, left for manual admin cleanup - nothing safe to rename them to). Idempotent across repeated -RC runs: already-renamed instances match on the second pass and are skipped. """ from app_data.data import Instance from app_data.fieldtypes.field_ipv4 import check_valid_ipv4_range from app_data.keyname import rename message = "migrate ipam_subnet instances to dataformat keyname" scanned = 0 migrated = 0 skipped = 0 for instance in Instance.stream_classname(classname="ipam_subnet"): scanned += 1 old_keyname = instance.keyname cidr = instance.get_attribute_first("subnet") if not cidr or cidr == old_keyname: continue if not check_valid_ipv4_range(cidr, strict=True): logger.warning( f" ipam_subnet/{old_keyname}: subnet field {cidr!r} isn't a " "valid aligned CIDR - left unmigrated, needs manual fix" ) skipped += 1 continue report = rename( classname="ipam_subnet", oldkeyname=old_keyname, newkeyname=cidr, dryrun=False ) if report["errors"]: logger.warning(f" ipam_subnet/{old_keyname} -> {cidr}: {report['errors']}") skipped += 1 continue migrated += 1 logger.info(f" OK - {message} ({migrated} migrated, {skipped} skipped, {scanned} scanned)")