Linkml conversion tooling

dandischema/models_importstab.py

@@ -0,0 +1,9 @@
+from .models_linkml import *  # noqa: F401,F403
+from .models_orig import DANDI_INSTANCE_URL_PATTERN  # noqa: F401
+
+# TODO: temporary imports of consts etc which might need to be 'redone'
+# so we do not duplicate them
+
+
+# TODO: do the extra tune ups like linking extra validations etc,
+# potentially copied from models_orig.py

dandischema/models_merge.yaml

@@ -0,0 +1,7 @@
+# This file specifies a partial schema to be merged with the auto LinkML translation
+# `dandischema.models`. It is a place to specify elements of the target dandischema
+# LinkML schema that are not translatable from or not available in `dandischema.models`
+
+slots:
+  schemaKey:
+    designates_type: true

dandischema/models_overlay.yaml

@@ -0,0 +1,31 @@
+name: dandi-schema
+id: https://schema.dandiarchive.org/s/dandi/v0.7
+version: 0.7.0
+status: eunal:concept-status/DRAFT
+
+prefixes:
+  dandiasset: http://dandiarchive.org/asset/
+  DANDI: http://dandiarchive.org/dandiset/
+  dandi: http://schema.dandiarchive.org/
+  dcite: http://schema.dandiarchive.org/datacite/
+  dct: http://purl.org/dc/terms/
+  linkml: https://w3id.org/linkml/
+  nidm: http://purl.org/nidash/nidm#
+  ORCID: https://orcid.org/
+  owl: http://www.w3.org/2002/07/owl#
+  PATO: http://purl.obolibrary.org/obo/PATO_
+  pav: http://purl.org/pav/
+  prov: http://www.w3.org/ns/prov#
+  rdfa: http://www.w3.org/ns/rdfa#
+  rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns#
+  rdfs: http://www.w3.org/2000/01/rdf-schema#
+  ROR: https://ror.org/
+  RRID: "https://scicrunch.org/resolver/RRID:"
+  rs: http://schema.repronim.org/
+  schema: http://schema.org/
+  skos: http://www.w3.org/2004/02/skos/core#
+  spdx: http://spdx.org/licenses/
+  uuid: http://uuid.repronim.org/
+  xsd: http://www.w3.org/2001/XMLSchema#
+
+default_prefix: dandi

pyproject.toml

@@ -62,6 +62,19 @@ all = [
 installer = "uv"
 python = "3.10"
 
+# Env for auto converting `dandischema.models` to LinkML schema and back to Pydantic models
+# To invoke conversion scripts provided by the environment, execute the following respective commands:
+#   `hatch run linkml-auto-converted:2linkml`    # Convert dandischema.models -> dandischema/models.yaml
+#   `hatch run linkml-auto-converted:2pydantic`  # Convert dandischema/models.yaml -> dandischema/models_linkml.py
+[tool.hatch.envs.linkml-auto-converted]
+dependencies = [
+    "black",  # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting
+    "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git"
+]
+[tool.hatch.envs.linkml-auto-converted.scripts]
+2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml"
+2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -e 's,dciteCOLON,,g' > dandischema/models_linkml.py"
+
 [tool.setuptools.packages.find]
 namespaces = true
 include = ["dandischema*"]

tools/linkml_conversion

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Script which will force us to apply the changes we have
+#
+
+set -eu
+
+# ensure we are clean to no side effects and are in
+test -z "$(git status --porcelain)" || { echo "Git tree is dirty, aborting."; exit 1; }
+
+# determine remote name based on what remote linkml-conversion is
+# tracked from which should correspond to our main repo!
+remote=$(git config branch.linkml-conversion.remote)
+
+git checkout linkml-conversion
+# ensure up to date!
+git pull --ff-only
+
+ver=$(git describe linkml-conversion)
+
+master_mergebase=$(git merge-base $remote/master linkml-conversion)
+
+git checkout linkml-auto-converted
+
+git merge -s ours --no-commit linkml-conversion
+
+git read-tree -m -u linkml-conversion
+
+# Poor man patch queue implementation
+# Edit this list if you want to merge or drop PRs branches to be patched with.
+# Order matters
+branches_to_merge=(
+    master
+    remove-discriminated-unions
+)
+
+echo "Applying patches from base $master_mergebase"
+
+for b in ${branches_to_merge[@]}; do
+    b_ver=$(git describe "$remote"/$b)
+    echo "Applying branch $b patch with following differences"
+    git diff --stat "$master_mergebase"..."$remote"/$b
+
+    git diff "$master_mergebase"..."$remote"/$b | patch -p1
+done
+
+hatch run linkml-auto-converted:2linkml
+hatch run linkml-auto-converted:2pydantic
+
+# add (re)generated files
+git add dandischema/models_linkml.py dandischema/models.yaml
+
+# add our stab and rename original models.py
+git mv dandischema/models.py dandischema/models_orig.py
+git mv dandischema/models_importstab.py dandischema/models.py
+
+# because we have pre-commit.ci doing that behind our back too!
+pre-commit run --all || :
+
+git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches" -a --no-verify