(wip) new sync service

appinfo/info.xml

@@ -34,6 +34,12 @@ Core App of the full-text search framework for your Nextcloud.
 		<job>OCA\FullTextSearch\Cron\Index</job>
 	</background-jobs>
 
+	<repair-steps>
+		<install>
+			<step>OCA\FullTextSearch\RepairStep\AppEnabled</step>
+		</install>
+	</repair-steps>
+
 	<commands>
 		<command>OCA\FullTextSearch\Command\Check</command>
 		<command>OCA\FullTextSearch\Command\CollectionInit</command>
@@ -50,6 +56,7 @@ Core App of the full-text search framework for your Nextcloud.
 		<command>OCA\FullTextSearch\Command\Reset</command>
 		<command>OCA\FullTextSearch\Command\Search</command>
 		<command>OCA\FullTextSearch\Command\Stop</command>
+		<command>OCA\FullTextSearch\Command\Sync</command>
 		<command>OCA\FullTextSearch\Command\Test</command>
 	</commands>
 

lib/AppInfo/Application.php

@@ -11,10 +11,14 @@
 
 
 use Closure;
+use NCU\FullTextSearch\IManager;
 use OC;
+use OC\FullTextSearch\FullTextSearchManager;
 use OCA\FullTextSearch\Capabilities;
 use OCA\FullTextSearch\ConfigLexicon;
+use OCA\FullTextSearch\Provider\FilesContentProvider;
 use OCA\FullTextSearch\Search\UnifiedSearchProvider;
+use OCA\FullTextSearch\Service\FullTextSearchService;
 use OCA\FullTextSearch\Service\IndexService;
 use OCA\FullTextSearch\Service\ProviderService;
 use OCA\FullTextSearch\Service\SearchService;
@@ -41,6 +45,9 @@ public function register(IRegistrationContext $context): void {
 		$context->registerCapability(Capabilities::class);
 		$context->registerSearchProvider(UnifiedSearchProvider::class);
 		$context->registerConfigLexicon(ConfigLexicon::class);
+
+		$context->registerFullTextSearchService(FullTextSearchService::class);
+
 		$this->registerServices($this->getContainer());
 	}
 
@@ -53,11 +60,8 @@ public function boot(IBootContext $context): void {
 		$context->injectFn(Closure::fromCallable([$this, 'registerNavigation']));
 	}
 
-
 	/**
-	 * Register Navigation Tab
-	 *
-	 * @param ContainerInterface $container
+	 * @deprecated
 	 */
 	protected function registerServices(ContainerInterface $container) {
 		/** @var IFullTextSearchManager $fullTextSearchManager */

lib/Command/Stop.php

@@ -41,7 +41,7 @@ protected function configure() {
 	protected function execute(InputInterface $input, OutputInterface $output) {
 		$output->writeln('stopping all running indexes');
 
-		$this->runningService->forceStop();
+		$this->runningService->stop();
 
 		return 0;
 	}

lib/Command/Sync.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\FullTextSearch\Command;
+
+use OC\Core\Command\Base;
+use OCA\FullTextSearch\Exceptions\PlatformTemporaryException;
+use OCA\FullTextSearch\Service\FullTextSearchService;
+use OCA\FullTextSearch\Service\LockService;
+use OCA\FullTextSearch\Service\LoggerService;
+use OCA\FullTextSearch\Service\SyncService;
+use OCA\FullTextSearch\Tools\Traits\TArrayTools;
+use Exception;
+use OC\Core\Command\InterruptedException;
+use OCA\FullTextSearch\ACommandBase;
+use OCA\FullTextSearch\Exceptions\TickDoesNotExistException;
+use OCA\FullTextSearch\Model\Index as ModelIndex;
+use OCA\FullTextSearch\Model\Runner;
+use OCA\FullTextSearch\Service\CliService;
+use OCA\FullTextSearch\Service\ConfigService;
+use OCA\FullTextSearch\Service\IndexService;
+use OCA\FullTextSearch\Service\PlatformService;
+use OCA\FullTextSearch\Service\ProviderService;
+use OCA\FullTextSearch\Service\RunningService;
+use OCP\IUserManager;
+use OutOfBoundsException;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Formatter\OutputFormatterStyle;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Terminal;
+use Throwable;
+
+class Sync extends Base {
+	public function __construct(
+		private readonly LockService $lockService,
+		private readonly SyncService $syncService,
+		private readonly FullTextSearchService $fullTextSearchService,
+		private readonly LoggerService $loggerService,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		parent::configure();
+		$this->setName('fulltextsearch:sync')
+			 ->setDescription('Index files')
+			->addOption('info', '', InputOption::VALUE_NONE, 'display info entries')
+			 ->addOption('no-output', '', InputOption::VALUE_NONE, 'no output, use nextcloud logs');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		if (!$input->getOption('no-output')) {
+			$this->loggerService->setOutputInterface($output, $input->getOption('info'));
+		}
+//		$this->fullTextSearchService->requestIndex('files', '123');
+		$this->lockService->lock();
+
+		while (true) {
+			try {
+				$this->lockService->update();
+				$this->syncProcess();
+			} catch (Exception $e) {
+				$this->loggerService->error('Exception while running fulltextsearch:sync', ['exception' => $e]);
+			}
+			sleep(10);
+		}
+
+		return 0;
+	}
+
+	private function syncProcess() {
+		$this->loggerService->info('initiating a new sync session');
+		$this->syncService->smartSync();
+		$this->loggerService->info('sync session closed');
+	}
+}
+

lib/ConfigLexicon.php

@@ -26,8 +26,10 @@ class ConfigLexicon implements ILexicon {
 	public const COLLECTION_INDEXING_LIST = 'collection_indexing_list';
 	public const COLLECTION_INTERNAL = 'collection_internal';
 	public const COLLECTION_LINKS = 'collection_links';
+	public const SYNC_REQUIREMENT_LEVEL = 'sync_requirement';
 	public const LOCK_ID = 'lock_id';
 	public const LOCK_PING = 'lock_ping';
+	public const ENABLED_SINCE = 'enabled_since';
 
 	public function getStrictness(): Strictness {
 		return Strictness::NOTICE;
@@ -42,9 +44,10 @@ public function getAppConfigs(): array {
 			new Entry(key: self::COLLECTION_INDEXING_LIST, type: ValueType::INT, defaultRaw: 50, definition: 'size of chunks of async documents on collection queue request'),
 			new Entry(key: self::COLLECTION_INTERNAL, type: ValueType::STRING, defaultRaw: 'local', definition: 'name of the local collection'),
 			new Entry(key: self::COLLECTION_LINKS, type: ValueType::ARRAY, defaultRaw: [], definition: '(internal) data relative to collections'),
-			// IAppConfig::FLAG_INTERNAL)
-			new Entry(key: self::LOCK_ID, type: ValueType::STRING, defaultRaw: '', definition: 'internal lock id', lazy: true),
-			new Entry(key: self::LOCK_PING, type: ValueType::INT, defaultRaw: 0, definition: 'internal lock time', lazy: true),
+			new Entry(key: self::SYNC_REQUIREMENT_LEVEL, type: ValueType::INT, defaultRaw: 0, definition: 'requirement level to confirm the need for sync'),
+			new Entry(key: self::LOCK_ID, type: ValueType::STRING, defaultRaw: '', definition: 'internal lock id', lazy: true, flags: IAppConfig::FLAG_INTERNAL),
+			new Entry(key: self::LOCK_PING, type: ValueType::INT, defaultRaw: 0, definition: 'internal lock time', lazy: true, flags: IAppConfig::FLAG_INTERNAL),
+			new Entry(key: self::ENABLED_SINCE, type: ValueType::INT, defaultRaw: 0, definition: 'the time since the fulltextsearch app is enabled', lazy: true, flags: IAppConfig::FLAG_INTERNAL),
 		];
 	}
 

lib/Db/SyncMapper.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\FullTextSearch\Db;
+
+use OCA\FullTextSearch\Model\DocumentSync;
+use OCP\AppFramework\Db\Entity;
+use OCP\AppFramework\Db\QBMapper;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+
+/**
+ * @template-extends QBMapper<DocumentSync>
+ */
+class SyncMapper extends QBMapper {
+	public const TABLE = 'fulltextsearch_sync';
+
+	public function __construct(
+		IDBConnection $db,
+	) {
+		parent::__construct($db, self::TABLE, DocumentSync::class);
+	}
+
+	/**
+	 * @return DocumentSync[]
+	 */
+	public function getForcedSyncs(int $limit = 100): array {
+		$qb = $this->db->getQueryBuilder();
+		$qb->select('*')
+		   ->from($this->getTableName())
+		   ->where($qb->expr()->eq('indexed', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+		   ->setMaxResults($limit);
+
+		return $this->findEntities($qb);
+	}
+
+	public function update(Entity $entity): Entity {
+		$qb = $this->db->getQueryBuilder();
+		$qb->update($this->getTableName())
+			->set('indexed', $qb->createNamedParameter($entity->getIndexed(), IQueryBuilder::PARAM_INT))
+			->where(
+				$qb->expr()->eq('provider_id', $qb->createNamedParameter($entity->getProviderId())),
+				$qb->expr()->eq('document_id', $qb->createNamedParameter($entity->getDocumentId())),
+			);
+		$qb->executeStatement();
+		return $entity;
+	}
+
+//	public function reset(string $providerId, string $documentId): void {
+//		$qb = $this->db->getQueryBuilder();
+//		$qb->update($this->getTableName())
+//		   ->set('indexed', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
+//		   ->where(
+//			   $qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId())),
+//			   $qb->expr()->eq('document_id', $qb->createNamedParameter($documentId())),
+//		   );
+//		$qb->executeStatement();
+//	}
+}

lib/Enum/SessionType.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\FullTextSearch\Enum;
+
+enum SessionType: string {
+	case UNKNOWN = '';
+	case FORCED = 'forced';
+	case SYNC = 'sync';
+	case RESYNC = 'sync_recent';
+}

lib/Migration/Version23001Date20220408140253.php

@@ -28,11 +28,11 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
 		/** @var ISchemaWrapper $schema */
 		$schema = $schemaClosure();
 
-		if (!$schema->hasTable('fulltextsearch_indexes')) {
+		if (!$schema->hasTable('fulltextsearch_sync')) {
 			return null;
 		}
 
-		$table = $schema->getTable('fulltextsearch_indexes');
+		$table = $schema->getTable('fulltextsearch_sync');
 		$column = $table->getColumn('message');
 
 		if ($column->getType()->getName() === Types::TEXT) {

lib/Migration/Version33001Date202511271645.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\FullTextSearch\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\Attributes\CreateTable;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+#[CreateTable('fulltextsearch_sync', ['provider_id', 'document_id', 'indexed'], description: 'table to manage indexed documents and new index request')]
+class Version33001Date202511271645 extends SimpleMigrationStep {
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+		if ($schema->hasTable('fulltextsearch_sync')) {
+			return null;
+		}
+
+		$table = $schema->createTable('fulltextsearch_sync');
+		$table->addColumn('provider_id', Types::STRING, [
+			'length' => 31,
+			'notnull' => true,
+		]);
+		$table->addColumn('document_id', Types::STRING, [
+			'length' => 31,
+			'notnull' => true,
+		]);
+		$table->addColumn('flags', Types::INTEGER, [
+			'length' => 7,
+			'default' => 0,
+		]);
+		$table->addColumn('indexed', Types::BIGINT, [
+			'length' => 11,
+			'default' => 0,
+		]);
+		$table->addColumn('checksum', Types::STRING, [
+			'length' => 16,
+			'notnull' => true,
+		]);
+
+		$table->setPrimaryKey(['provider_id', 'document_id'], 'fts_i_pd');
+		$table->addIndex(['indexed'], 'fts_i_i');
+
+		return $schema;
+	}
+}

lib/Model/DocumentSync.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OCA\FullTextSearch\Model;
+
+use OCP\AppFramework\Db\Entity;
+
+/**
+ * @method void setProviderId(string $providerId)
+ * @method string getProviderId()
+ * @method void setDocumentId(string $documentId)
+ * @method string getDocumentId()
+ * @method void setFlags(int $indexed)
+ * @method int getFlags()
+ * @method void setIndexed(int $indexed)
+ * @method int getIndexed()
+ * @method void setChecksum(string $checksum)
+ * @method string getChecksum()
+ * @psalm-suppress PropertyNotSetInConstructor
+ */
+class DocumentSync extends Entity {
+	protected string $providerId = '';
+	protected string $documentId = '';
+	protected int $flags = 0;
+	protected int $indexed = 0;
+	protected string $checksum = '';
+
+	public function __construct(
+	) {
+		$this->addType('providerId', 'string');
+		$this->addType('documentId', 'string');
+		$this->addType('flags', 'integer');
+		$this->addType('indexed', 'integer');
+		$this->addType('checksum', 'string');
+	}
+
+	public function definition(): string {
+		return $this->getProviderId() . '/' . $this->getDocumentId();
+	}
+}