diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 84bc8afdf..c8dfc391e 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -343,7 +343,7 @@ export const fetchNodeTypeSchemasForInstances = async ({ .select("source_local_id, schema_id") .eq("space_id", spaceId) .eq("is_schema", false) - .eq("arity", 0) + .eq("is_relation", false) .in("source_local_id", nodeInstanceIds); if (instanceError || !instanceRows) { @@ -365,7 +365,7 @@ export const fetchNodeTypeSchemasForInstances = async ({ .select("id, source_local_id, name") .eq("space_id", spaceId) .eq("is_schema", true) - .eq("arity", 0) + .eq("is_relation", false) .in("id", schemaIds); if (schemaError || !schemaRows) { diff --git a/apps/obsidian/src/utils/importRelations.ts b/apps/obsidian/src/utils/importRelations.ts index a3efd01e1..27911e315 100644 --- a/apps/obsidian/src/utils/importRelations.ts +++ b/apps/obsidian/src/utils/importRelations.ts @@ -171,7 +171,7 @@ const findOrCreateTriple = async ({ /** * Fetch relation instances from a remote space. Relation instances are concepts with - * is_schema=false and schema_id pointing to a relation type (arity=2). + * is_schema=false and schema_id pointing to a relation type */ export const fetchRelationInstancesFromSpace = async ({ client, @@ -187,7 +187,7 @@ export const fetchRelationInstancesFromSpace = async ({ ) .eq("space_id", spaceId) .eq("is_schema", false) - .gt("arity", 0); + .eq("is_relation", true); if (error || !instances) { console.warn("Error fetching relation instances:", error); diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index e18cfecf4..d81ca72cc 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -281,7 +281,7 @@ export const ensurePublishedRelationsAccuracy = async ({ .select("source_local_id") .eq("space_id", context.spaceId) .eq("is_schema", false) - .gt("arity", 0); + .eq("is_relation", true); if (syncedRelationIdsResult.error) { console.error( "Could not get synced relation ids", diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 9eb73e40a..e8021f346 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -184,7 +184,7 @@ const getLastNodeSchemaSyncTime = async ( .select("last_modified") .eq("space_id", spaceId) .eq("is_schema", true) - .eq("arity", 0) + .eq("is_relation", false) .order("last_modified", { ascending: false }) .limit(1) .maybeSingle(); @@ -200,7 +200,7 @@ const getLastRelationSchemaSyncTime = async ( .select("last_modified") .eq("space_id", spaceId) .eq("is_schema", true) - .gt("arity", 0) + .eq("is_relation", true) .order("last_modified", { ascending: false }) .limit(1) .maybeSingle(); @@ -216,7 +216,7 @@ const getLastRelationSyncTime = async ( .select("last_modified") .eq("space_id", spaceId) .eq("is_schema", false) - .gt("arity", 0) + .eq("is_relation", true) .order("last_modified", { ascending: false }) .limit(1) .maybeSingle(); @@ -350,7 +350,7 @@ const buildChangedNodesFromNodes = async ({ .from("my_concepts") .select("source_local_id") .eq("space_id", context.spaceId) - .eq("arity", 0) + .eq("is_relation", false) .eq("is_schema", false) .order("id"), 1000, @@ -517,7 +517,7 @@ const convertDgToSupabaseConcepts = async ({ .from("my_concepts") .select("source_local_id,literal_content") .eq("is_schema", true) - .eq("arity", 0) + .eq("is_relation", false) .eq("space_id", context.spaceId) .is("literal_content->>template_content", null); // could not filter on only absent keys, this includes nulls diff --git a/apps/obsidian/src/utils/templateImport.ts b/apps/obsidian/src/utils/templateImport.ts index 36b6daceb..db92593ba 100644 --- a/apps/obsidian/src/utils/templateImport.ts +++ b/apps/obsidian/src/utils/templateImport.ts @@ -90,7 +90,7 @@ export const fetchTemplateImportCandidates = async ({ "id, source_local_id, name, literal_content, author_id, space_id, last_modified", ) .eq("is_schema", true) - .eq("arity", 0) + .eq("is_relation", false) .eq("name", trimmedNodeTypeName) .neq("space_id", context.spaceId); diff --git a/apps/roam/src/utils/cleanupOrphanedNodes.ts b/apps/roam/src/utils/cleanupOrphanedNodes.ts index 69da5814f..7b9e3d69a 100644 --- a/apps/roam/src/utils/cleanupOrphanedNodes.ts +++ b/apps/roam/src/utils/cleanupOrphanedNodes.ts @@ -108,7 +108,7 @@ const getAllNodeSchemasFromSupabase = async ( .select("source_local_id") .eq("space_id", spaceId) .eq("is_schema", true) - .eq("arity", 0) + .eq("is_relation", false) .not("source_local_id", "is", null); if (error) { diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index 5427398e9..af20f4566 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -891,7 +891,7 @@ const getAllMissingOrNewDiscourseNodes = async ({ .from("my_concepts") .select("source_local_id") .eq("space_id", spaceId) - .eq("arity", 0) + .eq("is_relation", false) .eq("is_schema", false) .order("id"), 1000, @@ -938,7 +938,7 @@ const getSharedNodeInstanceSourceLocalIds = async ({ .select("source_local_id") .eq("space_id", spaceId) .eq("is_schema", false) - .eq("arity", 0) + .eq("is_relation", false) .order("source_local_id"), 1000, ); diff --git a/packages/database/doc/concept_design.md b/packages/database/doc/concept_design.md index f006842d4..601d8778c 100644 --- a/packages/database/doc/concept_design.md +++ b/packages/database/doc/concept_design.md @@ -23,7 +23,7 @@ Note: Implementors can skip this section. This generalization is rooted in prior experience with RDF, TopicMaps, semantic Frames, and David Spivak's [algebraic database](https://arxiv.org/abs/1602.03501) representation. In all cases, any type is defined by the attributes it can have. RDF calls them properties, Minsky's frames uses the term slots, etc. RDF/OWL distinguishes literal attributes (`owl: DatatypeProperty`, stored in the `literal_content` column) from attributes which are references to other objects (`owl:ObjectProperty` stored in the `reference_content` column). We adopted the term roles for ObjectProperties from TopicMaps. Spivak's work gives this distinction a categorical grounding. -Unifying relations and nodes in particular allows relations to both have extra attributes (as in a PropertyGraph vs. a strict RDF graph) and be referred to (either as the source/destination of another Relation, or as a target of an `ObjectProperty`.) Cliff Joslyn described this recursive mathematical structure as a [übergraph](https://arxiv.org/abs/1704.05547v1). It can also be expressed as reified relations in RDF-\*. +Unifying relations and nodes in particular allows non-intrinsic relations to both have extra attributes (as in a PropertyGraph vs. a strict RDF graph) and be referred to (either as the source/destination of another Relation, or as a target of an `ObjectProperty`.) Cliff Joslyn described this recursive mathematical structure as a [übergraph](https://arxiv.org/abs/1704.05547v1). It can also be expressed as reified relations in RDF-\*. ### Use cases @@ -49,6 +49,7 @@ Base: CrossAppNodeSchema: # Base and... + slotDefinitions: Record # Forthcoming metadata?: Json label: string template?: string @@ -80,6 +81,7 @@ CrossAppNode: nodeType: LocalId content.direct: InlineCrossAppContent content.full?: InlineCrossAppTypedContent + slots: Record # Forthcoming CrossAppRelation: # Base and... @@ -102,28 +104,37 @@ CrossAppRelation: | `authorId` | <-> | `author_local_id` => `author_id` | | `metadata` | <- | `literal_content` | -Note on metadata: It is read from the database, but currently not written. +Note on metadata: It is read from the database, but currently not written. We will phase it out. + When some specific keys of `literal_content` are mapped to a CrossApp field, those key-value pairs are not included again in the `metadata`. Thus `metadata` is a grab-bag for residual data. (Eg color for now.) Residual (not otherwise accounted for) keys in Obsidian frontmatter are mapped to `literal_content->source_data`. (Not through CrossApp.) #### Matching of CrossAppNodeSchema and Concept -| CrossAppNodeSchema | Concept | value | -| ------------------ | ----------------------------------- | ----- | -| `label` | `name` | {} | -| `template` | `literal_content->template_content` | | -| `templateTitle` | `literal_content->template` | | -| - | `is_schema` | true | -| - | `schema_id` | null | -| - | `arity` | 0 | -| - | `reference_content` | {} | - -Note that this does not yet allow for ObjectProperties to be defined; this would require to define `literal_content->roles`, corresponding `reference_content`s, and change the `arity`. See below. +| CrossAppNodeSchema | Concept | value | +| ---------------------- | ----------------------------------- | ----- | +| `label` | `name` | {} | +| `template` | `literal_content->template_content` | | +| `templateTitle` | `literal_content->template` | | +| - | `is_schema` | true | +| - | `schema_id` | null | +| - | `arity` | 0 | +| - | `is_relation` | false | +| `slotDefinition` | `reference_content` | | +| `keys(slotDefinition)` | `literal_content->roles` | | + +In the case of a node with ObjectRelations, such as Evidence, we would see something like: + +| CrossAppNode | Concept | value | +| ---------------------- | ------------------------ | ------------------------------------- | +| - | `arity` | 1 | +| `slotDefinitions` | `reference_content` | {"sourceDocument": } | +| `keys(slotDefinition)` | `literal_content->roles` | {"roles":["sourceDocument"]} | Also note: In Obsidian, where we do not go through CrossAppNodeSchema, the label is also assigned to `literal_content->label`. -Query filter: `.eq("arity",0).eq("is_schema", true)` +Query filter: `.eq("is_relation", false).eq("is_schema", true)` #### Matching of CrossAppNode and Concept @@ -131,11 +142,21 @@ Query filter: `.eq("arity",0).eq("is_schema", true)` | ----------------------- | ----------------------------------------------- | ------------------ | | `content->direct->text` | `name` | | | - | `is_schema` | false | -| - | `arity` | 0 | -| - | `reference_content` | {} | +| - | `is_relation` | false | | `nodeType` | `schema_represented_by_local_id` => `schema_id` | ref to Node schema | +| - | `arity` | 0 | +| `slots` | `reference_content` | {} | + +In the case of a node with ObjectRelations, such as Evidence, we would see: + +| CrossAppNode | Concept | value | +| ------------ | ------------------- | ------------------------ | +| - | `arity` | 1 | +| `slots` | `reference_content` | {"sourceDocument": } | -Query filter: `.eq("arity",0).eq("is_schema", false)` +The keys of the slots (instance variables) should match those defined in the `literal_content->roles` of the corresponding schema, as shown above. + +Query filter: `.eq("is_relation", false).eq("is_schema", false)` #### Matching of CrossAppRelationTypeSchema and Concept (Obsidian only) @@ -148,8 +169,9 @@ Query filter: `.eq("arity",0).eq("is_schema", false)` | - | `reference_content` | {} | | `label` | `name`, `literal_content->label` | | | `complement` | `literal_content->complement` | | +| - | `is_relation` | true | -Query filter: `.eq("arity",2).eq("is_schema", true).is("reference_content->source", "null")` +Query filter: `.eq("is_relation", true).eq("is_schema", true).is("reference_content->source", "null")` #### Matching of Obsidian CrossAppRelationTripleSchema and Concept @@ -165,10 +187,13 @@ Query filter: `.eq("arity",2).eq("is_schema", true).is("reference_content->sourc | `sourceType` | `reference_content->source` | ref to Node schema | | `destinationType` | `reference_content->destination` | ref to Node schema | | `relation` | `reference_content->relation_type` | ref to RelationType schema | +| - | `is_relation` | true | + +Note that the relationType in `reference_content->relation_type` is not here playing the role of a slot definition, as it will not be defined in relation instances; so it does not belong in the roles. It can be thought of as an instance variable of the `CrossAppRelationTripleSchema` class itself. -Note that putting the relationType in `reference_content->relation_type` without a corresponding role was a hackish shortcut, and should be revisited (see below.) +Theory: If we were materialize five metaclasses for the five kinds of objects, we would define the `relation_type` as a role (`slotDefinition`) of the `CrossAppRelationTripleSchema` metaclass, and its range constraint would have to point for a metaclass corresponding to `RelationTypeSchema`. It may help conceptually, but there is no reason to materialize those metaclasses in the database. -Query filter: `.eq("arity",2).eq("is_schema", true).not("reference_content->relation_type", "is", "null")` +Query filter: `.eq("is_relation", true).eq("is_schema", true).not("reference_content->relation_type", "is", "null")` #### Matching of Roam CrossAppRelationTripleSchema and Concept @@ -183,12 +208,13 @@ Query filter: `.eq("arity",2).eq("is_schema", true).not("reference_content->rela | `complement` | `literal_content->complement` | | | `sourceType` | `reference_content->source` | ref to Node schema | | `destinationType` | `reference_content->destination` | ref to Node schema | +| - | `is_relation` | true | -Query filter: `.eq("arity",2).eq("is_schema", true).not("reference_content->source", "is", "null").is("reference_content->relation_type", "null")` +Query filter: `.eq("is_relation", true).eq("is_schema", true).not("reference_content->source", "is", "null").is("reference_content->relation_type", "null")` In most cases, you would want both Roam and Obsidian RelationTripleSchemas, hence you would simply use: -Combined query filter: `.eq("arity",2).eq("is_schema", true).not("reference_content->source", "is", "null")` +Combined query filter: `.eq("is_relation", true).eq("is_schema", true)` #### Matching of Obsidian CrossAppRelation and Concept @@ -201,22 +227,20 @@ Combined query filter: `.eq("arity",2).eq("is_schema", true).not("reference_cont | `relationType` | `schema_id` (in Roam) | ref to RelationTriple schema | | `source` | `reference_content->source` | ref to Node | | `destination` | `reference_content->destination` | ref to Node | +| - | `is_relation` | true | -Query filter: `.eq("arity",2).eq("is_schema", false)` +Again, the keys of the slots match those defined in the `literal_content->roles` of the corresponding schema. In the case of relations, those must include `source` and `destination`. Those slot definitions are implicit. We may allow extra slot definitions for relationTypeSchema some day, but those would be besides the implicit `source` and `destination`. -### Design considerations and future changes +Query filter: `.eq("is_relation", true).eq("is_schema", false)` -#### Arity and ObjectProperties - -The arity on either a schema or instance is based on the size of the `literal_content->roles` array in the schema. -The `reference_content` allows either single or multiple values (`Record`). The `reference_content` values are collated in a computed column `refs`, whose index allows for efficient filters on sql queries before digging into the `reference_content` jsonb. +### Design considerations and future changes -As a first approximation, we distinguished relations from nodes using `arity==2`, but this precludes using ObjectProperties. +#### Relations, arity and ObjectProperties -To remedy this, we propose adding a computed column `is_relation`, which would check whether (source, destination) are both in the schema's roles. The query filters using `arity` would be redefined to use `is_relation`. +The arity on either a schema or instance is based on the size of the `literal_content->roles` array in the schema. Think of the `roles` literal as a class variable that defines which variables will exist in instances. -Internal node references such as the Evidence's Source can then be expressed using roles and internal relations. +The `reference_content` allows either single or multiple values (`Record`). The `reference_content` values are collated in a computed column `refs`, whose index allows for efficient filters on sql queries before digging into the `reference_content` jsonb. Currently, the slots only accommodate single values. -In `RelationTripleSchema`, we refer to the `RelationTypeSchema` with a `relation_type` entry in the `reference_content` column. This, unusually, is not backed by an entry in the `roles`. This is a deviation from the mental model, but in the current situation, adding that role would break the `arity=2` checks. Introducing the `is_relation` column will also allow this to be part of the roles. +Relations are concepts (schemas or instances) whose schemas have the special roles `source` and `destination`. This is captured in a computed column `is_relation`. -Note that we probably won't add a range constraint in that case; the constraint should require the `relation_type` to be any `RelationTypeSchema`, but there is no row materializing this meta-class. +Internal node references such as the Evidence's Source can be expressed using a slotDefinition which will show up in the schema's roles and slots which will show up in the instances' `reference_content`. diff --git a/packages/database/schema.puml b/packages/database/schema.puml index 9cac789fd..a010ca2a6 100644 --- a/packages/database/schema.puml +++ b/packages/database/schema.puml @@ -59,6 +59,7 @@ class "Concept" [[{An abstract concept, claim or relation}]] { {field} reference_content : JSON {field} literal_content : JSON {field} is_schema : boolean + {field} is_relation : boolean } "LocalAccess" --> "0..1" "Space" : "space" "Content" --> "0..1" "Space" : "space" @@ -104,6 +105,7 @@ class "ConceptSchema" [[{A Concept that describes a schema (type) for other conc {field} reference_content(i) : JSON {field} literal_content(i) : JSON {field} is_schema(i) : boolean + {field} is_relation(i) : boolean } "Concept" --> "1" "ConceptSchema" : "schema" "Concept" ^-- "ConceptSchema" diff --git a/packages/database/schema.svg b/packages/database/schema.svg index ab85a5238..a743feee0 100644 --- a/packages/database/schema.svg +++ b/packages/database/schema.svg @@ -1 +1 @@ -PlatformAccountid : integername : stringplatform : Platformaccount_local_id : stringwrite_permission : booleanactive : booleanagent_type : AgentTypemetadata : JSONUsersid : integerSpaceAccesseditor : booleanSpaceid : integerurl : stringname : stringplatform : PlatformLocalAccessContentid : integersource_local_id : stringcreated : datetimetext : stringmetadata : JSONscale : Scalelast_modified : datetimeDocumentid : integersource_local_id : stringcreated : datetimemetadata : JSONlast_modified : datetimeurl : stringcontents : blobConceptid : integerepistemic_status : EpistemicStatusname : stringdescription : stringcreated : datetimelast_modified : datetimearity : integerreference_content : JSONliteral_content : JSONis_schema : booleanAgentIdentifieridentifier_type : AgentIdentifierTypevalue : stringtrusted : booleanContentEmbeddingmodel : EmbeddingNamevector : vectorobsolete : booleanContentEmbedding_openai_text_embedding_3_small_1536model(i) : EmbeddingNamevector(i) : vectorobsolete(i) : booleanConceptSchemaid(i) : integerepistemic_status(i) : EpistemicStatusname(i) : stringdescription(i) : stringcreated(i) : datetimelast_modified(i) : datetimearity(i) : integerreference_content(i) : JSONliteral_content(i) : JSONis_schema(i) : booleandg_account0..1account1space0..1space0..1space0..1space0..1space0..1account1author1contributors0..*creator1author1contributors0..*author1contributors0..*account1document1target1part_of0..1represented_by0..1schema1 \ No newline at end of file +PlatformAccountid : integername : stringplatform : Platformaccount_local_id : stringwrite_permission : booleanactive : booleanagent_type : AgentTypemetadata : JSONUsersid : integerSpaceAccesseditor : booleanSpaceid : integerurl : stringname : stringplatform : PlatformLocalAccessContentid : integersource_local_id : stringcreated : datetimetext : stringmetadata : JSONscale : Scalelast_modified : datetimeDocumentid : integersource_local_id : stringcreated : datetimemetadata : JSONlast_modified : datetimeurl : stringcontents : blobConceptid : integerepistemic_status : EpistemicStatusname : stringdescription : stringcreated : datetimelast_modified : datetimearity : integerreference_content : JSONliteral_content : JSONis_schema : booleanis_relation : booleanAgentIdentifieridentifier_type : AgentIdentifierTypevalue : stringtrusted : booleanContentEmbeddingmodel : EmbeddingNamevector : vectorobsolete : booleanContentEmbedding_openai_text_embedding_3_small_1536model(i) : EmbeddingNamevector(i) : vectorobsolete(i) : booleanConceptSchemaid(i) : integerepistemic_status(i) : EpistemicStatusname(i) : stringdescription(i) : stringcreated(i) : datetimelast_modified(i) : datetimearity(i) : integerreference_content(i) : JSONliteral_content(i) : JSONis_schema(i) : booleanis_relation(i) : booleandg_account0..1account1space0..1space0..1space0..1space0..1space0..1account1author1contributors0..*creator1author1contributors0..*author1contributors0..*account1document1target1part_of0..1represented_by0..1schema1 \ No newline at end of file diff --git a/packages/database/schema.yaml b/packages/database/schema.yaml index 5f44eef21..b8d4d174e 100644 --- a/packages/database/schema.yaml +++ b/packages/database/schema.yaml @@ -333,7 +333,7 @@ classes: range: integer required: true ifabsent: "0" - description: The number of roles in this relation; nodes have zero, binary relations have 2, etc. + description: The number of roles in this relation schema: range: ConceptSchema required: true @@ -345,12 +345,17 @@ classes: literal_content: range: JSON required: true - description: "Aspects of the concept that have literal values. `{[key: string]: any}`" + description: "Aspects of the concept that have literal values. `{[key: string]: any}`. For schemas, may include a list of `roles` for instances." ifabsent: "{}" is_schema: range: boolean required: true ifabsent: "false" + is_relation: + range: boolean + required: true + description: "Whether the concept acts as a relation, i.e. has `source` and `destination` among its roles or its schema's roles" + ifabsent: "false" represented_by: description: This concept is explicitly represented by a given content unit range: Content diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index 76d7fa84b..995d16658 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -331,9 +331,9 @@ const composeConceptQuery = ({ } let query = supabase.from("my_concepts").select(q); if (scope.type === "nodes") { - query = query.eq("arity", 0); + query = query.eq("is_relation", false); } else if (scope.type === "relations") { - query = query.gt("arity", 0); + query = query.eq("is_relation", true); } // else fetch both @@ -544,6 +544,7 @@ export const CONCEPT_FIELDS: (keyof Concept)[] = [ "last_modified", "space_id", "arity", + "is_relation", "literal_content", "reference_content", "refs", @@ -682,7 +683,7 @@ export const getNodesByType = async ({ }; /** - * Retrieves all discourse relations (concepts with arity > 0) in a space. + * Retrieves all discourse relations in a space. * * @param params - Query parameters * @param params.supabase - Authenticated Supabase client diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts index f7c36aaf5..36dd023e7 100644 --- a/packages/database/src/lib/sharedNodes.ts +++ b/packages/database/src/lib/sharedNodes.ts @@ -221,7 +221,7 @@ const getSharedNodeRows = async ({ .select(CONCEPT_COLUMNS) .neq("space_id", currentSpaceId) .eq("is_schema", false) - .eq("arity", 0), + .eq("is_relation", false), client .from("my_contents") .select(DIRECT_CONTENT_COLUMNS) @@ -282,7 +282,7 @@ export const getSharedNodeByRid = async ({ .eq("space_id", space.id) .eq("source_local_id", sourceLocalId) .eq("is_schema", false) - .eq("arity", 0), + .eq("is_relation", false), client .from("my_contents") .select(DIRECT_CONTENT_COLUMNS)