From 66430289d0425894e0a05938857a1a74b3a2a752 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:34:56 +0200 Subject: [PATCH 1/3] improve purge --- README.md | 13 +-- .../Console/Command/DeletedProjectsPurge.php | 96 +++++++++++-------- 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index e854e60..cdd3e5b 100644 --- a/README.md +++ b/README.md @@ -240,20 +240,11 @@ Run command: Use number of days or 0 as show to remove expiration completely. By default, it's dry-run. Override with `-f` parameter. ### Purge deleted projects -Purge already deleted projects (remove residual metadata, optionally ignoring backend errors) using a Manage API token and a CSV piped via STDIN. +Purge already deleted projects (remove residual metadata, optionally ignoring backend errors) using a Manage API token. ``` -cat deleted-projects.csv | php cli.php storage:deleted-projects-purge [--ignore-backend-errors] +php cli.php storage:deleted-projects-purge [--ignore-backend-errors] [--ignore-backend-errors] ``` -Input CSV header must be exactly: -``` -id,name -``` -Behavior: -- Validates header. -- For each row calls Manage API purgeDeletedProject; prints command execution id. -- Polls every second (max 600s) until project `isPurged` is true; errors on timeout. -- With --ignore-backend-errors it instructs API to ignore backend failures and just purge metadata (buckets/workspaces records). ### Set data retention for multiple projects Set data retention days for specific projects listed in a CSV piped via STDIN. diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index 0448db5..2cf02fc 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -1,7 +1,9 @@ setName('storage:deleted-projects-purge') ->setDescription('Purge deleted projects.') + ->addArgument('url', InputArgument::REQUIRED, 'URL of stack including https://') ->addArgument('token', InputArgument::REQUIRED, 'manage api token') + ->addArgument('projectIds', InputArgument::IS_ARRAY | InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') ->addOption('ignore-backend-errors', null, InputOption::VALUE_NONE, "Ignore errors from backend and just delete buckets and workspaces metadata") - ; + ->addOption('force', null, InputOption::VALUE_NONE, 'Actually perform destructive operations (purge). Without this flag, the command will only simulate actions.'); } - protected function execute(InputInterface $input, OutputInterface $output) { + $url = $input->getArgument('url'); $token = $input->getArgument('token'); - - $fh = fopen('php://stdin', 'r'); - if (!$fh) { - throw new \Exception('Error on input read'); - } - + $projectIds = $input->getArgument('projectIds'); $ignoreBackendErrors = (bool) $input->getOption('ignore-backend-errors'); + $force = (bool) $input->getOption('force'); $output->writeln(sprintf( 'Ignore backend errors %s', $ignoreBackendErrors ? 'On' : 'Off' )); + $output->writeln(sprintf( + 'Force mode %s', + $force ? 'On (destructive operations will be performed)' : 'Off (no destructive operations will be performed)' + )); $client = new Client([ + 'url' => $url, 'token' => $token, ]); - $lineNumber = 0; - while ($row = fgetcsv($fh)) { - if ($lineNumber === 0) { - $this->validateHeader($row); - } else { - $this->purgeProject( - $client, - $output, - $ignoreBackendErrors, - $row[0], - $row[1] - ); - } - $lineNumber++; + foreach ($projectIds as $projectId) { + $this->purgeProject( + $client, + $output, + $ignoreBackendErrors, + (int) $projectId, + $force, + ); } } - private function validateHeader($header) - { - $expectedHeader = ['id', 'name']; - if ($header !== $expectedHeader) { - throw new \Exception(sprintf( - 'Invalid input header: %s Expected header: %s', - implode(',', $header), - implode(',', $expectedHeader) - )); + private function purgeProject( + Client $client, + OutputInterface $output, + bool $ignoreBackendErrors, + int $projectId, + bool $force, + ): void { + try { + $deletedProject = $client->getDeletedProject($projectId); + if ($deletedProject['isPurged'] === true) { + $output->writeln(sprintf('INFO Project "%d" purged already.', $projectId)); + return; + } + } catch (ClientException $e) { + if ($e->getCode() === 404) { + $output->writeln(sprintf('Error: Purge of the project "%d" not found.', $projectId)); + + return; + } + $output->writeln(sprintf('Error: Purge of the project "%d" is not possible due "%s".', $projectId, $e->getMessage())); + return; } - } - private function purgeProject(Client $client, OutputInterface $output, $ignoreBackendErrors, $projectId, $projectName) - { + $projectName = $deletedProject['name'] ?? 'unknown'; $output->writeln(sprintf('Purge %s (%d)', $projectName, $projectId)); + if (!$force) { + $output->writeln("[DRY-RUN] Would purge project $projectId"); + return; + } + $response = $client->purgeDeletedProject($projectId, [ - 'ignoreBackendErrors' => (bool) $ignoreBackendErrors, + 'ignoreBackendErrors' => $ignoreBackendErrors, ]); $output->writeln(" - execution id {$response['commandExecutionId']}"); @@ -87,9 +100,16 @@ private function purgeProject(Client $client, OutputInterface $output, $ignoreBa if (time() - $startTime > $maxWaitTimeSeconds) { throw new \Exception("Project {$projectId} purge timeout."); } - sleep(1); + sleep(2); + $output->writeln( + sprintf(' - - Waiting for project "%s" (%s) to be purged: execution id %s', + $projectName, + $projectId, + $response['commandExecutionId'], + ) + ); } while ($deletedProject['isPurged'] !== true); - $output->writeln(sprintf('Purge done %s (%d)', $projectName, $projectId)); + $output->writeln(sprintf('Purge done "%s" (%d)', $projectName, $projectId)); } } From d5b0202ecda62c08900e73186dcff81cf86c42a1 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:37:16 +0200 Subject: [PATCH 2/3] cs --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index 2cf02fc..fbc0501 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -102,7 +102,8 @@ private function purgeProject( } sleep(2); $output->writeln( - sprintf(' - - Waiting for project "%s" (%s) to be purged: execution id %s', + sprintf( + ' - - Waiting for project "%s" (%s) to be purged: execution id %s', $projectName, $projectId, $response['commandExecutionId'], From dfd77d67cacbe8fe4f9dee258f16fc36d65f9e51 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:41:48 +0200 Subject: [PATCH 3/3] explode ids --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index fbc0501..075651b 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -19,7 +19,7 @@ protected function configure() ->setDescription('Purge deleted projects.') ->addArgument('url', InputArgument::REQUIRED, 'URL of stack including https://') ->addArgument('token', InputArgument::REQUIRED, 'manage api token') - ->addArgument('projectIds', InputArgument::IS_ARRAY | InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') + ->addArgument('projectIds', InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') ->addOption('ignore-backend-errors', null, InputOption::VALUE_NONE, "Ignore errors from backend and just delete buckets and workspaces metadata") ->addOption('force', null, InputOption::VALUE_NONE, 'Actually perform destructive operations (purge). Without this flag, the command will only simulate actions.'); } @@ -45,6 +45,7 @@ protected function execute(InputInterface $input, OutputInterface $output) 'url' => $url, 'token' => $token, ]); + $projectIds = array_filter(explode(',', $projectIds), 'is_numeric'); foreach ($projectIds as $projectId) { $this->purgeProject(