diff --git a/src/nostrdb.c b/src/nostrdb.c index b7ae80835..144784b3d 100644 --- a/src/nostrdb.c +++ b/src/nostrdb.c @@ -65,13 +65,6 @@ typedef int (*ndb_migrate_fn)(struct ndb_txn *); typedef int (*ndb_word_parser_fn)(void *, const char *word, int word_len, int word_index); -/* parsed nip10 reply data */ -struct ndb_note_reply { - unsigned char *root; - unsigned char *reply; - unsigned char *mention; -}; - // these must be byte-aligned, they are directly accessing the serialized data // representation #pragma pack(push, 1) @@ -2030,17 +2023,18 @@ static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type) } /* get reply information from a note */ -static void ndb_parse_reply(struct ndb_note *note, struct ndb_note_reply *note_reply) +void ndb_note_get_reply(struct ndb_note *note, struct ndb_note_reply *note_reply) { unsigned char *root, *reply, *mention, *id; const char *marker; struct ndb_iterator iter; struct ndb_str str; uint16_t count; - int any_marker, first; + int any_marker, first, have_explicit_root, marker_len; any_marker = 0; first = 1; + have_explicit_root = 0; root = NULL; reply = NULL; mention = NULL; @@ -2052,13 +2046,26 @@ static void ndb_parse_reply(struct ndb_note *note, struct ndb_note_reply *note_r break; marker = NULL; + marker_len = 0; count = ndb_tag_count(iter.tag); if (count < 2) continue; str = ndb_tag_str(note, iter.tag, 0); - if (!(str.flag == NDB_PACKED_STR && str.str[0] == 'e')) + if (str.flag != NDB_PACKED_STR) + continue; + + if (str.str[0] == 'E') { + str = ndb_tag_str(note, iter.tag, 1); + if (str.flag == NDB_PACKED_ID) { + root = str.id; + have_explicit_root = 1; + } + continue; + } + + if (str.str[0] != 'e') continue; str = ndb_tag_str(note, iter.tag, 1); @@ -2068,25 +2075,41 @@ static void ndb_parse_reply(struct ndb_note *note, struct ndb_note_reply *note_r /* if we have the marker, assign it */ if (count >= 4) { - str = ndb_tag_str(note, iter.tag, 3); - if (str.flag == NDB_PACKED_STR) - marker = str.str; + struct ndb_str marker_str = ndb_tag_str(note, iter.tag, 3); + if (marker_str.flag != NDB_PACKED_ID) { + marker = marker_str.str; + marker_len = ndb_str_len(&marker_str); + } } if (marker) { any_marker = true; - if (!strcmp(marker, "root")) + if (marker_len == 4 && !memcmp(marker, "root", 4)) root = id; - else if (!strcmp(marker, "reply")) + else if (marker_len == 5 && !memcmp(marker, "reply", 5)) reply = id; - else if (!strcmp(marker, "mention")) + else if (marker_len == 7 && !memcmp(marker, "mention", 7)) mention = id; - } else if (!any_marker && first) { - root = id; + continue; + } + + if (any_marker) + continue; + + if (first) { first = 0; - } else if (!any_marker && !reply) { - reply = id; + if (!have_explicit_root) { + root = id; + continue; + } + if (!reply) + reply = id; + continue; } + + if (!reply) + reply = id; + } note_reply->reply = reply; @@ -2094,7 +2117,7 @@ static void ndb_parse_reply(struct ndb_note *note, struct ndb_note_reply *note_r note_reply->mention = mention; } -static int ndb_is_reply_to_root(struct ndb_note_reply *reply) +int ndb_note_reply_is_to_root(struct ndb_note_reply *reply) { if (reply->root && !reply->reply) return 1; @@ -2151,12 +2174,12 @@ int ndb_count_replies(struct ndb_txn *txn, const unsigned char *note_id, uint16_ break; if (!(note = ndb_get_note_by_key(txn, note_key, &size))) continue; - if (ndb_note_kind(note) != 1) + if (ndb_note_kind(note) != 1 && ndb_note_kind(note) != 1111) continue; - ndb_parse_reply(note, &reply); + ndb_note_get_reply(note, &reply); - if (ndb_is_reply_to_root(&reply)) { + if (ndb_note_reply_is_to_root(&reply)) { reply_id = reply.root; } else { reply_id = reply.reply; @@ -6070,8 +6093,8 @@ static void ndb_process_note_stats( ndb_increment_quote_metadata(txn, quoted_note_id, scratch, scratch_size); } - ndb_parse_reply(note, &reply); - if (ndb_is_reply_to_root(&reply)) { + ndb_note_get_reply(note, &reply); + if (ndb_note_reply_is_to_root(&reply)) { reply_id = reply.root; } else { reply_id = reply.reply; @@ -6133,7 +6156,7 @@ static uint64_t ndb_write_note(struct ndb_txn *txn, ndb_write_note_relay_indexes(txn, &relay_key); // only parse content and do fulltext index on text and longform notes - if (kind == 1 || kind == 30023) { + if (kind == 1 || kind == 30023 || kind == 1111) { if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_FULLTEXT)) { if (!ndb_write_note_fulltext_index(txn, note->note, note_key)) return 0; @@ -8987,6 +9010,7 @@ enum ndb_common_kind ndb_kind_to_common_kind(int kind) { case 0: return NDB_CKIND_PROFILE; case 1: return NDB_CKIND_TEXT; + case 1111: return NDB_CKIND_COMMENT; case 3: return NDB_CKIND_CONTACTS; case 4: return NDB_CKIND_DM; case 5: return NDB_CKIND_DELETE; @@ -9010,6 +9034,7 @@ const char *ndb_kind_name(enum ndb_common_kind ck) switch (ck) { case NDB_CKIND_PROFILE: return "profile"; case NDB_CKIND_TEXT: return "text"; + case NDB_CKIND_COMMENT: return "comment"; case NDB_CKIND_CONTACTS: return "contacts"; case NDB_CKIND_DM: return "dm"; case NDB_CKIND_DELETE: return "delete"; diff --git a/src/nostrdb.h b/src/nostrdb.h index a9e6c2a0f..06b7dec53 100644 --- a/src/nostrdb.h +++ b/src/nostrdb.h @@ -252,6 +252,7 @@ enum ndb_dbs { enum ndb_common_kind { NDB_CKIND_PROFILE, NDB_CKIND_TEXT, + NDB_CKIND_COMMENT, NDB_CKIND_CONTACTS, NDB_CKIND_DM, NDB_CKIND_DELETE, @@ -691,6 +692,12 @@ int ndb_stat(struct ndb *ndb, struct ndb_stat *stat); void ndb_stat_counts_init(struct ndb_stat_counts *counts); // NOTE +struct ndb_note_reply { + unsigned char *root; + unsigned char *reply; + unsigned char *mention; +}; + const char *ndb_note_content(struct ndb_note *note); struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr); uint32_t ndb_note_content_length(struct ndb_note *note); @@ -702,6 +709,8 @@ unsigned char *ndb_note_sig(struct ndb_note *note); void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind); struct ndb_tags *ndb_note_tags(struct ndb_note *note); int ndb_str_len(struct ndb_str *str); +void ndb_note_get_reply(struct ndb_note *note, struct ndb_note_reply *reply); +int ndb_note_reply_is_to_root(struct ndb_note_reply *reply); /// write the note as json to a buffer int ndb_note_json(struct ndb_note *, char *buf, int buflen); diff --git a/test.c b/test.c index 1916b5721..7e76ea0af 100644 --- a/test.c +++ b/test.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) @@ -30,6 +32,53 @@ static void delete_test_db() { unlink(TEST_DIR "/data.lock"); } +static void assert_optional_id_eq(unsigned char *actual, const char *expected_hex) +{ + if (!expected_hex) { + assert(actual == NULL); + return; + } + + assert(actual != NULL); + + char actual_hex[65]; + char expected_lower[65]; + size_t expected_len = strlen(expected_hex); + + assert(hex_encode(actual, 32, actual_hex)); + actual_hex[64] = '\0'; + + assert(expected_len == 64); + + for (size_t i = 0; i < expected_len; i++) { + char c = expected_hex[i]; + expected_lower[i] = (char)tolower((unsigned char)c); + } + expected_lower[expected_len] = '\0'; + + if (strcmp(actual_hex, expected_lower)) { + fprintf(stderr, "expected id %s but got %s\n", expected_lower, actual_hex); + } + + assert(strcmp(actual_hex, expected_lower) == 0); +} + +static unsigned char *nip10_effective_reply(struct ndb_note_reply *reply) +{ + if (reply->reply) + return reply->reply; + if (reply->root) + return reply->root; + return NULL; +} + +static unsigned char *nip10_reply_to_root(struct ndb_note_reply *reply) +{ + if (ndb_note_reply_is_to_root(reply)) + return reply->root; + return NULL; +} + int ndb_rebuild_reaction_metadata(struct ndb_txn *txn, const unsigned char *note_id, struct ndb_note_meta_builder *builder, uint32_t *count); int ndb_count_replies(struct ndb_txn *txn, const unsigned char *note_id, uint16_t *direct_replies, uint32_t *thread_replies); @@ -81,6 +130,8 @@ static void test_count_metadata() struct ndb_note_meta_entry *entry; uint16_t count, direct_replies[2]; uint32_t total_reactions, reactions, thread_replies[2]; + const uint16_t expected_direct_replies = 59; + const uint32_t expected_thread_replies = 99; int i; reactions = 0; @@ -121,11 +172,11 @@ static void test_count_metadata() thread_replies[0] = *ndb_note_meta_counts_thread_replies(entry); printf("\t# thread replies %d\n", thread_replies[0]); - assert(thread_replies[0] == 93); + assert(thread_replies[0] == expected_thread_replies); direct_replies[0] = *ndb_note_meta_counts_direct_replies(entry); printf("\t# direct replies %d\n", direct_replies[0]); - assert(direct_replies[0] == 83); + assert(direct_replies[0] == expected_direct_replies); total_reactions = *ndb_note_meta_counts_total_reactions(entry); printf("\t# total reactions %d\n", reactions); @@ -2039,6 +2090,120 @@ static void test_note_relay_index() printf("ok test_note_relay_index\n"); } +static void test_nip10_marker(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"id\":\"19377cb4b9b807561830ab6d4c1fae7b9c9f1b623c15d10590cacc859cf19d76\",\"pubkey\":\"4871687b7b0aee3f1649c866e61724d79d51e673936a5378f5ed90bf7580791f\",\"created_at\":1714170678,\"kind\":1,\"tags\":[[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3\",\"\",\"reply\"],[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4\",\"wss://relay.damus.io\",\"root\"]],\"content\":\"hi\",\"sig\":\"53921b1572c2e4373180a9f71513a0dee286cba6193d983052f96285c08f0e0158773d82ac97991ba8d390f6f54f84d5272c2e945f2e854a750f9cf038c0f759\"}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4"); + assert_optional_id_eq(reply.reply, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(reply.mention, NULL); + assert(ndb_note_reply_is_to_root(&reply) == 0); + assert_optional_id_eq(nip10_effective_reply(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(nip10_reply_to_root(&reply), NULL); + + printf("ok test_nip10_marker\n"); +} + +static void test_nip10_deprecated(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"id\":\"ebac7df823ab975b6d2696505cf22a959067b74b1761c5581156f2a884036997\",\"pubkey\":\"118758f9a951c923b8502cfb8b2f329bee2a46356b6fc4f65c1b9b4730e0e9e5\",\"created_at\":1714175831,\"kind\":1,\"tags\":[[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4\"],[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3\"]],\"content\":\"hi\",\"sig\":\"05913c7b19a70188d4dec5ac53d5da39fea4d5030c28176e52abb211e1bde60c5947aca8af359a00c8df8d96127b2f945af31f21fe01392b661bae12e7d14b1d\"}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4"); + assert_optional_id_eq(reply.reply, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(reply.mention, NULL); + assert(ndb_note_reply_is_to_root(&reply) == 0); + assert_optional_id_eq(nip10_effective_reply(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(nip10_reply_to_root(&reply), NULL); + + printf("ok test_nip10_deprecated\n"); +} + +static void test_nip10_mention(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"id\":\"9521de81704269f9f61c042355eaa97a845a90c0ce6637b290800fa5a3c0b48d\",\"pubkey\":\"b3aceb5b36a235377c80dc2a1b3594a1d49e394b4d74fa11bc7cb4cf0bf677b2\",\"created_at\":1714177990,\"kind\":1,\"tags\":[[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3\",\"\",\"mention\"],[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4\",\"wss://relay.damus.io\",\"root\"]],\"content\":\"hi\",\"sig\":\"e908ec395f6ea907a4b562b3ebf1bf61653566a5648574a1f8c752285797e5870e57416a0be933ce580fc3d65c874909c9dacbd1575c15bd97b8a68ea2b5160b\"}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4"); + assert_optional_id_eq(reply.reply, NULL); + assert_optional_id_eq(reply.mention, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert(ndb_note_reply_is_to_root(&reply) == 1); + assert_optional_id_eq(nip10_effective_reply(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4"); + assert_optional_id_eq(nip10_reply_to_root(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d4"); + + printf("ok test_nip10_mention\n"); +} + +static void test_nip10_marker_mixed(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"content\":\"Go to pleblab plz\",\"created_at\":1714157088,\"id\":\"19ae8cd276185f6f48fd7e25736c260ea0ac25d9b591ec3194631e3196e19622\",\"kind\":1,\"pubkey\":\"ae1008d23930b776c18092f6eab41e4b09fcf3f03f3641b1b4e6ee3aa166d760\",\"sig\":\"fdafc7192a0f3b5fef5ae794ef61eb2b3c7cc70bace53f3aa6d4263347581d36add7e9468a4e329d9c986e3a5c46e4689a6b79f60c5cf7778a403316ac5b2629\",\"tags\":[[\"e\",\"27e71cf53299dafb5dc7bcc0a078357418a4375cb1097bf5184662493f79a627\",\"\",\"root\"],[\"e\",\"f99046bd87be7508d55e139de48517c06ef90830d77a5d3213df858d77bb2f8f\"],[\"e\",\"1a616998552cf76e9786f76ac68f6104cdae46377330735c68bfe0b9426d2fa8\",\"\",\"reply\"],[\"p\",\"3efdaebb1d8923ebd99c9e7ace3b4194ab45512e2be79c1b7d68d9243e0d2681\"],[\"p\",\"8ea485266b2285463b13bf835907161c22bb3da1e652b443db14f9cee6720a43\"],[\"p\",\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\"]]}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "27e71cf53299dafb5dc7bcc0a078357418a4375cb1097bf5184662493f79a627"); + assert_optional_id_eq(reply.reply, "1a616998552cf76e9786f76ac68f6104cdae46377330735c68bfe0b9426d2fa8"); + assert_optional_id_eq(reply.mention, NULL); + assert(ndb_note_reply_is_to_root(&reply) == 0); + assert_optional_id_eq(nip10_effective_reply(&reply), "1a616998552cf76e9786f76ac68f6104cdae46377330735c68bfe0b9426d2fa8"); + assert_optional_id_eq(nip10_reply_to_root(&reply), NULL); + + printf("ok test_nip10_marker_mixed\n"); +} + +static void test_nip10_reply_to_root_with_reply_tag(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"id\":\"22c4986d970bb13a9337bdad4e462bc75c5105375669d87caeab0951e76af800\",\"pubkey\":\"592295cf2b09a7f9555f43adb734cbee8a84ee892ed3f9336e6a09b6413a0db9\",\"created_at\":1753380428,\"kind\":1,\"tags\":[[\"e\",\"343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0\",\"ws://relay.jb55.com/\",\"root\",\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\"],[\"e\",\"343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0\",\"ws://relay.jb55.com/\",\"reply\",\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\"],[\"p\",\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\",\"ws://relay.jb55.com/\"]],\"content\":\"So the user part is like nip-C0\",\"sig\":\"d2224177462f3cadfba2ab946005deb3f7485232a9aed78e5304a9f96a1170b45d48c925686293a0272c661db287e201924d49f1216d402fd1f34aa57da70b60\"}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0"); + assert_optional_id_eq(reply.reply, "343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0"); + assert_optional_id_eq(reply.mention, NULL); + assert(ndb_note_reply_is_to_root(&reply) == 1); + assert_optional_id_eq(nip10_effective_reply(&reply), "343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0"); + assert_optional_id_eq(nip10_reply_to_root(&reply), "343ff2fe97e352c7012a44dc85135dccef43acb73e459e71f7284c9627b57ab0"); + + printf("ok test_nip10_reply_to_root_with_reply_tag\n"); +} + +static void test_nip10_deprecated_reply_to_root(void) +{ + struct ndb_note_reply reply; + struct ndb_note *note; + const char *event = "{\"id\":\"140280b7886c48bddd99684b951c6bb61bebc8270a4989f316282c72aa35e5ba\",\"pubkey\":\"5ee7067e7155a9abf494e3e47e3249254cf95389a0c6e4f75cbbf35c8c675c23\",\"created_at\":1714178274,\"kind\":1,\"tags\":[[\"e\",\"7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3\"]],\"content\":\"hi\",\"sig\":\"e433d468d49fbc0f466b1a8ccefda71b0e17af471e579b56b8ce36477c116109c44d1065103ed6c01f838af92a13e51969d3b458f69c09b6f12785bd07053eb5\"}"; + unsigned char buffer[4096]; + + assert(ndb_note_from_json(event, strlen(event), ¬e, buffer, sizeof(buffer))); + ndb_note_get_reply(note, &reply); + assert_optional_id_eq(reply.root, "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(reply.reply, NULL); + assert_optional_id_eq(reply.mention, NULL); + assert(ndb_note_reply_is_to_root(&reply) == 1); + assert_optional_id_eq(nip10_effective_reply(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + assert_optional_id_eq(nip10_reply_to_root(&reply), "7d33c272a74e75c7328b891ab69420dd820cc7544fc65cd29a058c3495fd27d3"); + + printf("ok test_nip10_deprecated_reply_to_root\n"); +} + static void test_nip50_profile_search() { struct ndb *ndb; struct ndb_txn txn; @@ -2228,6 +2393,12 @@ int main(int argc, const char *argv[]) { test_replay_attack(); test_custom_filter(); + test_nip10_marker(); + test_nip10_deprecated(); + test_nip10_mention(); + test_nip10_marker_mixed(); + test_nip10_reply_to_root_with_reply_tag(); + test_nip10_deprecated_reply_to_root(); test_metadata(); test_count_metadata(); test_reaction_encoding(); @@ -2291,6 +2462,3 @@ int main(int argc, const char *argv[]) { printf("All tests passed!\n"); // Print this if all tests pass. } - - -