From 0ae91c9364a74a452c2c4c62c664720e3e4340f9 Mon Sep 17 00:00:00 2001 From: Nathon Fowlie <2831833+nathonfowlie@users.noreply.github.com> Date: Sun, 14 May 2023 12:55:09 +0000 Subject: [PATCH 1/4] improv: sync package version with git tag Updated pyproject.toml so that the closest git tag is always used to set the release version when the pip package is built. refs: - resolves #80 --- LICENSE | 21 ++++++++++++++++++++ pyproject.toml | 46 +++++++++++++++++++++++++++++++++++++++++++- src/rybo/__init__.py | 5 +++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 LICENSE create mode 100644 src/rybo/__init__.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c31efd8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Alex Laverty + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 50494a7..bf29102 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,47 @@ +[build-system] +requires = ["setuptools>=41", "wheel", "setuptools-git-versioning<2"] +build-backend = "setuptools.build_meta" + +[project] +name = "rybo" +dynamic = ["version"] +authors = [ + { name="Alex Laverty" } +] +maintainers = [ + { name="Alex Laverty" }, + { name="Nathon Fowlie" } +] +description="Reddit to YouTube bot." +readme="README.md" +requires-python=">=3.11" +classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent" +] +keywords = [ + "reddit", + "youtube", + "text-to-speech", + "bot" +] +license={file="LICENSE"} + +[project.urls] +"Homepage" = "https://github.com/alexlaverty/python-reddit-youtube-bot" +"Documentation" = "http://alexlaverty.github.io/python-reddit-youtube-bot" +"Repository" = "https://github.com/alexlaverty/python-reddit-youtube-bot.git" +"Bug Tracker" = "https://github.com/alexlaverty/python-reddit-youtube-bot/issues" + +[project.scripts] +rybo = "rybo.cli.rybo:cli" + [tool.flake8] -max-line-length = 88 +docstring-convention = 'google' +exclude = 'venv' extend-ignore = ['E203'] +max-line-length = 88 [tool.isort] profile = "black" @@ -11,3 +52,6 @@ max-line-length = 88 [tool.pycodestyle] max-line-length = 88 ignore = ['E203'] + +[tool.setuptools-git-versioning] +enabled = true \ No newline at end of file diff --git a/src/rybo/__init__.py b/src/rybo/__init__.py new file mode 100644 index 0000000..d56549f --- /dev/null +++ b/src/rybo/__init__.py @@ -0,0 +1,5 @@ +"""Reddit to YouTube bot.""" +import importlib.metadata + +# Use pyproject.toml as the single source of truth for the package version. +__version__ = importlib.metadata.version(__name__) From f09650245bcac1889816cba9a71dfc1ad9e689bf Mon Sep 17 00:00:00 2001 From: Nathon Fowlie <2831833+nathonfowlie@users.noreply.github.com> Date: Sun, 14 May 2023 23:34:55 +1000 Subject: [PATCH 2/4] fix: configuration issues on devcontainer Fixed configuration issues on the dev container that were causing the postCreateCommand script to bomb out. Once initialisation is complete, an editable version of the rybo utility will be installed inside the devcontainer, and can be called from the command line via `rybo`. Also added additional features to support code quality/consistency. refs: - resolves #81 --- .devcontainer/Dockerfile | 7 +++++++ .devcontainer/devcontainer.json | 22 +++++++++++++++++----- dependencies/scripts/postCreateCommand.sh | 10 ++++++++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index eca1caa..533681a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -29,6 +29,13 @@ RUN mkdir -p /home/vscode/.local/bin && \ chmod +x /home/vscode/.local/bin/yt-dlp && \ chown -R vscode /home/vscode/.local +# Enables preservation of bash history +RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" && \ + mkdir /commandhistory && \ + touch /commandhistory/.bash_history && \ + chown -R $USERNAME /commandhistory && \ + echo "$SNIPPET" >> "/home/$USERNAME/.bashrc" + USER $USERNAME ENTRYPOINT ["/bin/bash"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cdcbc83..73139e5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,6 @@ "python.pythonPath": "/usr/local/bin/python", "python.languageServer": "Pylance", "python.globalModuleInstallation": false, - "python.logging.level": "debug", "python.venvPath": "~/.pyenv", "python.terminal.activateEnvInCurrentTerminal": true, "python.analysis.autoSearchPaths": true, @@ -41,27 +40,40 @@ "terminal.integrated.gpuAcceleration": "auto", "terminal.integrated.useWslProfiles": true, "terminal.integrated.defaultProfile.linux":"bash" - } + }, + "extensions": [ + "ms-python.python" + ] }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", ] }, "features": { "ghcr.io/devcontainers/features/docker-outside-of-docker:1.2.1": {}, - "ghcr.io/devcontainers/features/git:1.1.5": {} + "ghcr.io/devcontainers/features/git:1.1.5": {}, + "ghcr.io/devcontainers-contrib/features/twine:2": {}, + "ghcr.io/devcontainers-contrib/features/mypy:2": {}, + "ghcr.io/devcontainers-contrib/features/isort:2": {}, + "ghcr.io/devcontainers-contrib/features/flake8:2": {}, + "ghcr.io/devcontainers-contrib/features/black:2": {}, + "ghcr.io/devcontainers-contrib/features/bandit:2": {} }, + "mounts": [ + "source=projectname-bashhistory,target=/commandhistory,type=volume" + ], + // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "/scripts/postCreateCommand.sh", + "postCreateCommand": "/bin/bash dependencies/scripts/postCreateCommand.sh", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" diff --git a/dependencies/scripts/postCreateCommand.sh b/dependencies/scripts/postCreateCommand.sh index f1c5dcc..c48380c 100644 --- a/dependencies/scripts/postCreateCommand.sh +++ b/dependencies/scripts/postCreateCommand.sh @@ -11,8 +11,14 @@ pip install --user -r requirements.txt # Additional dependencies needed to develop the utility & maintain code # quality. -pip install --user -r requirements-dev.txt +pip install --user -r requirements-dev.txt # Initialise playwright +playwright install playwright install-deps -playwright install firefox + +# Install an editable version of the rybo utility. Code changes will be applied +# in real-time. +pip install --editable . + +/bin/bash \ No newline at end of file From f3141260092805f5230a3d3ba21931a981c98f74 Mon Sep 17 00:00:00 2001 From: Nathon Fowlie <2831833+nathonfowlie@users.noreply.github.com> Date: Sun, 14 May 2023 23:41:55 +1000 Subject: [PATCH 3/4] fix: ignore egg packages Updated gitignore to ignore egg-info. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index edf10ce..1ad4096 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,9 @@ client_secret.json cookies*.json credentials.storage videos/ -config/auth.py .env +venv +*.egg-info **/__pycache__/ assets/work_dir/ \ No newline at end of file From 3be18cce6d7d0b98e16f9a5e192a80802ba5c6bb Mon Sep 17 00:00:00 2001 From: Nathon Fowlie <2831833+nathonfowlie@users.noreply.github.com> Date: Mon, 15 May 2023 01:55:27 +1000 Subject: [PATCH 4/4] feat: add cli wrapper Major change to allow the bot to be distributed as a pip package. app.py has been removed and replaced with a CLI utility called 'rybo'. Existing video generation, TTS and Reddit post processing logic has been left as is, however now sits inside the CLI. Removed auth.py, as credentials are now passed via environment variables or a YAML configuration file. The default expected location of the configuration file is `~/rybo.yaml`. A sample configuration file has been included in the codebase. ArgParse parameters updated to allow loading of configuration from the command line, a configuration file, or environment variables. The order of precendence for loading configuration options is: 1. Configuration file 2. Environment variables. 3. CLI parameters. Configuration options provided as CLI parameters will override the same option provided as an environment variable, and the environment variable will override the same option provided in the configuration file. Replaced all calls to "print" with module specific loggers to ensure consistent output and facilitate configurable logging behaviour. Updated existing GitHub workflows to reflect the new expected environment variable names. Updated the readme with instructions on how to use the new CLI. BREAKING CHANGE: app.py no longer exists, use the `rybo` command. refs: - resolves #82 --- .dockerignore | 1 - .github/workflows/pr.yml | 29 +- .github/workflows/tssvibelounge.yml | 32 +- README.md | 450 ++++++++++----- app.py | 369 ------------ config/auth-env.py | 18 - config/auth-example.py | 18 - {comments => cookies}/cookie-dark-mode.json | 0 {comments => cookies}/cookie-light-mode.json | 0 requirements-dev.txt | 2 +- requirements.txt | 3 + rybo.yaml | 82 +++ src/rybo/cli/__init__.py | 1 + src/rybo/cli/rybo.py | 525 ++++++++++++++++++ src/rybo/commands/__init__.py | 229 ++++++++ src/rybo/comments/__init__.py | 1 + {comments => src/rybo/comments}/screenshot.py | 68 ++- src/rybo/config/__init__.py | 1 + {config => src/rybo/config}/settings.py | 2 +- src/rybo/logging.py | 30 + src/rybo/publish/__init__.py | 1 + {publish => src/rybo/publish}/login.py | 45 +- {publish => src/rybo/publish}/upload.py | 289 +++++----- {publish => src/rybo/publish}/youtube.py | 29 +- src/rybo/reddit/__init__.py | 1 + {reddit => src/rybo/reddit}/reddit.py | 264 +++++---- src/rybo/speech/__init__.py | 1 + {speech => src/rybo/speech}/speech.py | 153 +++-- .../rybo/speech}/streamlabs_polly.py | 149 ++--- {speech => src/rybo/speech}/tiktok.py | 33 +- src/rybo/thumbnail/__init__.py | 1 + {thumbnail => src/rybo/thumbnail}/keywords.py | 15 +- {thumbnail => src/rybo/thumbnail}/lexica.py | 15 +- .../rybo/thumbnail}/thumbnail.py | 46 +- src/rybo/utils/__init__.py | 109 ++++ {utils => src/rybo/utils}/base64_encoding.py | 9 +- {utils => src/rybo/utils}/common.py | 11 +- csvmgr.py => src/rybo/utils/csvmgr.py | 14 +- {utils => src/rybo/utils}/speed_test.py | 0 src/rybo/video_generation/__init_-.py | 1 + .../rybo/video_generation}/video.py | 286 +++++----- 41 files changed, 2079 insertions(+), 1254 deletions(-) delete mode 100644 app.py delete mode 100644 config/auth-env.py delete mode 100644 config/auth-example.py rename {comments => cookies}/cookie-dark-mode.json (100%) rename {comments => cookies}/cookie-light-mode.json (100%) create mode 100644 rybo.yaml create mode 100644 src/rybo/cli/__init__.py create mode 100644 src/rybo/cli/rybo.py create mode 100644 src/rybo/commands/__init__.py create mode 100644 src/rybo/comments/__init__.py rename {comments => src/rybo/comments}/screenshot.py (81%) create mode 100644 src/rybo/config/__init__.py rename {config => src/rybo/config}/settings.py (100%) create mode 100644 src/rybo/logging.py create mode 100644 src/rybo/publish/__init__.py rename {publish => src/rybo/publish}/login.py (94%) rename {publish => src/rybo/publish}/upload.py (90%) rename {publish => src/rybo/publish}/youtube.py (78%) create mode 100644 src/rybo/reddit/__init__.py rename {reddit => src/rybo/reddit}/reddit.py (65%) create mode 100644 src/rybo/speech/__init__.py rename {speech => src/rybo/speech}/speech.py (86%) rename {speech => src/rybo/speech}/streamlabs_polly.py (94%) rename {speech => src/rybo/speech}/tiktok.py (95%) create mode 100644 src/rybo/thumbnail/__init__.py rename {thumbnail => src/rybo/thumbnail}/keywords.py (79%) rename {thumbnail => src/rybo/thumbnail}/lexica.py (85%) rename {thumbnail => src/rybo/thumbnail}/thumbnail.py (92%) create mode 100644 src/rybo/utils/__init__.py rename {utils => src/rybo/utils}/base64_encoding.py (86%) rename {utils => src/rybo/utils}/common.py (96%) rename csvmgr.py => src/rybo/utils/csvmgr.py (88%) rename {utils => src/rybo/utils}/speed_test.py (100%) create mode 100644 src/rybo/video_generation/__init_-.py rename {video_generation => src/rybo/video_generation}/video.py (75%) diff --git a/.dockerignore b/.dockerignore index 0180a78..d7ea92f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ final/* -config/auth.py .env client_secret.json cookies.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 22d6177..c9d9a21 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,16 +13,16 @@ jobs: - name: Run ttsvibelounge Script env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RYBO_POLLY_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + RYBO_POLLY_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} CREDENTIALS_STORAGE: ${{ secrets.CREDENTIALS_STORAGE }} - PRAW_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} - PRAW_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} - PRAW_USER_AGENT: ${{ secrets.PRAW_USER_AGENT }} - PRAW_USERNAME: ${{ secrets.PRAW_USERNAME }} - PRAW_PASSWORD: ${{ secrets.PRAW_PASSWORD }} - RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} - RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} + RYBO_REDDIT_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} + RYBO_REDDIT_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} + RYBO_REDDIT_USER_AGENT: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0 + RYBO_REDDIT_USERNAME: ${{ secrets.PRAW_USERNAME }} + RYBO_REDDIT_PASSWORD: ${{ secrets.PRAW_PASSWORD }} + RYBO_RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} + RYBO_RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} YOUTUBE_CLIENT_SECRET: ${{ secrets.YOUTUBE_CLIENT_SECRET }} GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | @@ -31,12 +31,13 @@ jobs: echo $GITHUB_WORKSPACE echo $YOUTUBE_CLIENT_SECRET > client_secret.json echo $CREDENTIALS_STORAGE > credentials.storage - cp config/auth-env.py config/auth.py + pip install -r requirements.txt playwright install - python3 app.py --total-posts 1 \ - --enable-background \ - --background-directory /app/assets/backgrounds \ - --enable-mentions + pip install --editable . + rybo --total-posts 1 \ + --enable-background \ + --background-directory /app/assets/backgrounds \ + --enable-mentions python3 refresh_token.py rm -f client_secret.json rm -f credentials.storage diff --git a/.github/workflows/tssvibelounge.yml b/.github/workflows/tssvibelounge.yml index e4141be..1d37de4 100644 --- a/.github/workflows/tssvibelounge.yml +++ b/.github/workflows/tssvibelounge.yml @@ -29,16 +29,16 @@ jobs: - name: Run ttsvibelounge Script env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RYBO_POLLY_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + RYBO_POLLY_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} CREDENTIALS_STORAGE: ${{ secrets.CREDENTIALS_STORAGE }} - PRAW_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} - PRAW_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} - PRAW_USER_AGENT: ${{ secrets.PRAW_USER_AGENT }} - PRAW_USERNAME: ${{ secrets.PRAW_USERNAME }} - PRAW_PASSWORD: ${{ secrets.PRAW_PASSWORD }} - RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} - RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} + RYBO_REDDIT_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} + RYBO_REDDIT_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} + RYBO_REDDIT_USER_AGENT: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0 + RYBO_REDDIT_USERNAME: ${{ secrets.PRAW_USERNAME }} + RYBO_REDDIT_PASSWORD: ${{ secrets.PRAW_PASSWORD }} + RYBO_RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} + RYBO_RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} YOUTUBE_CLIENT_SECRET: ${{ secrets.YOUTUBE_CLIENT_SECRET }} GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | @@ -47,18 +47,18 @@ jobs: echo $GITHUB_WORKSPACE echo $YOUTUBE_CLIENT_SECRET > client_secret.json echo $CREDENTIALS_STORAGE > credentials.storage - cp config/auth-env.py config/auth.py + pip install -r requirements.txt playwright install - python3 app.py --total-posts 1 \ - --enable-upload \ - --enable-background \ - --background-directory /app/assets/backgrounds \ - --enable-mentions + pip install --editable . + rybo --total-posts 1 \ + --enable-upload \ + --enable-background \ + --background-directory /app/assets/backgrounds \ + --enable-mentions python3 refresh_token.py rm -f client_secret.json rm -f credentials.storage - - name: check for changes run: | git config --global --add safe.directory $(realpath .) diff --git a/README.md b/README.md index fc1bb1e..10a4d9c 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,59 @@ # Automated Reddit to Youtube Bot -* [Description](#description) -* [Example Videos](#example-videos) -* [Install Prerequisite Components](#install-prerequisite-components) -* [Git clone repository](#git-clone-repository) -* [Generate Reddit Tokens](#generate-reddit-tokens) -* [Copy auth config](#copy-auth-config) -* [Python Pip Install Dependencies](#python-pip-install-dependencies) -* [Install Playwright](#install-playwright) -* [Run Python Script](#run-python-script) -* [Generate a Video for a Specific Post](#generate-a-video-for-a-specific-post) -* [Generate Only Thumbnails](#generate-only-thumbnails) -* [Enable a Newscaster](#enable-a-newscaster) -* [Settings.py File](#settings.py-file) - +* [Description](#Description) +* [Example Videos](#ExampleVideos) +* [Windows](#Windows) + * [ Install Prerequisite Components](#InstallPrerequisiteComponents) + * [ Clone the Git Repository](#ClonetheGitRepository) + * [ Configure a Virtual Environment](#ConfigureaVirtualEnvironment) + * [ Configure Playwright](#ConfigurePlaywright) + * [ Install the pip package](#Installthepippackage) + * [ Generate a Reddit token](#GenerateaReddittoken) + * [ Set up your credentials](#Setupyourcredentials) + * [ Run the CLI utility](#RuntheCLIutility) +* [Configuration](#Configuration) + * [Configuring the CLI](#ConfiguringtheCLI) + * [ Downloading video backgrounds using yt-dlp](#Downloadingvideobackgroundsusingyt-dlp) + * [ Help](#Help) +* [Customising](#Customising) + * [ Specify Subreddits to Scrape](#SpecifySubredditstoScrape) + * [ Exclude Subreddits](#ExcludeSubreddits) + * [ Filter Reddit submissions by keyword](#FilterRedditsubmissionsbykeyword) + * [ Change the Text to Speech Engine](#ChangetheTexttoSpeechEngine) + * [ Limit the number of generated videos](#Limitthenumberofgeneratedvideos) + * [ Skip Reddit posts by submission score](#SkipRedditpostsbysubmissionscore) + * [ Filter Reddit posts by title length](#FilterRedditpostsbytitlelength) + * [ Filter Reddit posts by self text length](#FilterRedditpostsbyselftextlength) + * [ Filter Reddit posts by comment count](#FilterRedditpostsbycommentcount) + * [Limit number of posts to process](#Limitnumberofpoststoprocess) + * [ Configure number of thumbnails to generate](#Configurenumberofthumbnailstogenerate) + * [ Set the maximum video length](#Setthemaximumvideolength) + * [ Set maximum number of comments to include](#Setmaximumnumberofcommentstoinclude) + * [ Specify folder paths](#Specifyfolderpaths) + * [ Set video dimensions](#Setvideodimensions) + * [ Skip video compilation](#Skipvideocompilation) + * [ Skip YouTube uploading](#SkipYouTubeuploading) + * [ Add a video overlay](#Addavideooverlay) + * [ Add a Newscaster](#AddaNewscaster) + * [ Add a pause after each TTS file](#AddapauseaftereachTTSfile) + * [Modify appearnace of text](#Modifyappearnaceoftext) + * [ Download images from Lexica](#DownloadimagesfromLexica) +* [ Tips and Tricks](#TipsandTricks) + * [Generate a Video for a Specific Post](#GenerateaVideoforaSpecificPost) + * [Generate Only Thumbnails](#GenerateOnlyThumbnails) + * [Enable a Newscaster](#EnableaNewscaster) -## Description +## Description Scrape posts from Reddit and automatically generate Youtube Videos and Thumbnails -## Example Videos +## Example Videos Checkout my Youtube Channel for example videos made by this repo : @@ -37,70 +65,150 @@ Checkout my Youtube Channel for example videos made by this repo : # Quickstart Guide -# Windows - +## Windows [Watch the Python Reddit Youtube Bot Tutorial Video :](https://youtu.be/LaFFU9EskfA) [![Watch the video](assets/images/python-reddit-youtube-bot-tutorial.png)](https://youtu.be/LaFFU9EskfA) -## Install Prerequisite Components +### Install Prerequisite Components Install these prerequisite components first : * Git - https://git-scm.com/download/win -* Python 3.10 - https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe +* Python 3.11 - https://www.python.org/ftp/python/3.11.3/python-3.11.3-amd64.exe * Microsoft C++ Build Tools - https://visualstudio.microsoft.com/visual-cpp-build-tools/ * ImageMagick - https://imagemagick.org/script/download.php#windows -## Git clone repository +### Clone the Git Repository -``` -git clone git@github.com:alexlaverty/python-reddit-youtube-bot.git -cd python-reddit-youtube-bot +```powershell +> git clone git@github.com:alexlaverty/python-reddit-youtube-bot.git +> cd python-reddit-youtube-bot ``` -## Generate Reddit Tokens +### Configure a Virtual Environment -Generate Reddit PRAW Tokens - https://www.reddit.com/prefs/apps/ +Create a virtual environment, and install package dependencies : -## Copy auth config +```powershell +> python -m venv venv +... +> .\venv\Scripts\activate.ps1 +... +> pip install -r requirements.txt +Collecting boto3==1.26.123 + Using cached boto3-1.26.123-py3-none-any.whl (135 kB) +Collecting bs4==0.0.1 + Using cached bs4-0.0.1.tar.gz (1.1 kB) + Preparing metadata (setup.py) ... done +... +``` -Create a copy of the auth-example.py file and name it auth.py : +### Configure Playwright -``` -copy config/auth-example.py config/auth.py +Install and configure playwright by running : + +```powershell +> playwright install ``` -Update the `auth.py` file to contain the Reddit Auth tokens you generated in the previous step. +### Install the pip package -## Python Pip Install Dependencies +Install the command-line utility used to run the bot : -``` -pip install -r requirements.txt +```powershell +> pip install --user --editable . +Obtaining file:///workspaces/python-reddit-youtube-bot-forked + ... +Building wheels for collected packages: rybo + ... +Successfully built rybo +Installing collected packages: rybo + ... +Successfully installed rybo-0.0.1 ``` -## Install Playwright +### Generate a Reddit token -Install and configure playwright by running : +Generate Reddit OAuth credentials via https://www.reddit.com/prefs/apps/. -``` -playwright install -``` +### Set up your credentials -## Run Python Script +Add the Reddit token via environment variables, or in a YAML configuration +file. The default expected location for this file is `$HOME/rybo.yaml`. -Run the python script : +| Environment Variable | Configuration File Option | Description | +|--------------------------------|---------------------------|------------------------------------------------------------| +| `RYBO_REDDIT_CLIENT_ID` | `reddit.client_id` | Client id used to authenticate against the Reddit API. | +| `RYBO_REDDIT_CLIENT_SECRET` | `reddit.client_secret` | Client secret used to authenticate against the Reddit API. | +| `RYBO_REDDIT_USERNAME` | `reddit.username` | Username used to log in to the Reddit Web UI. | +| `RYBO_REDDIT_PASSWORD` | `reddit.password` | Password used to log in to the Reddit Web UI. | +| `RYBO_POLLY_ACCESS_KEY` | `polly.access_key` | AWS Access Key used to interact with the Polly service. | +| `RYBO_POLLY_SECRET_ACCESS_KEY` | `polly.secret_access_key` | AWS secret used to interact with the Polly service. | +| `RYBO_RUMBLE_USERNAME` | `rumble.username` | Username used to interact with the Rumble Web UI. | +| `RYBO_RUMBLE_PASSWORD` | `rumble.password` | Password used to interact with the Rumble Web UI. | +### Run the CLI utility + +```powershell +> rybo ``` -python app.py -``` -when it completes the video will be generated into the `videos` folder and will be named `final.mp4` +When it completes, the video will be generated into the `videos` folder and +will be named `final.mp4`. + +## Configuration + +### Configuring the CLI + +Configuration options can be specified via CLI parameters, environment variables, or a +yaml configuration file. + +The order of precedence is as follows: -# Downloading video backgrounds using yt-dlp : +1. Yaml configuration file (default location: `$HOME/rybo.yaml`) +2. Environment variables. +3. User-specified CLI parameters. + +For example, if `disable_overlay: True` is set in `$HOME/rybo.yaml`, and +the `RYBO_DISABLE_OVERLAY=False` environment variable is also set, then +video overlay will be disabled (false) in the runtime configuration. + +
+Click to view the list of configuration options + +| Argument | Environment Variable | Configuration File Option | Default Value | Description | +|--------------------------|--------------------------------|---------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-h`/`--help` | | | | Display the help menu. | +| `--config` | `RYBO_CONFIG` | | `$HOME/rybo.yaml` | Load settings from a configuration file. | +| `--version ` | | | | Display version information. | +| `--background-directory` | `RYBO_BACKGROUND_DIRECTORY` | `background_directory` | `assets/backgrounds` | Folder path to videos that will be used for the video background. | +| `--comment-style` | `RYBO_COMMENT_STYLE` | `comment_style` | `reddit` | Use text based, or image based Reddit comments. Choices are **text** or **image**. | +| `--disable-overlay` | `RYBO_DISABLE_OVERLAY` | `disable_overlay` | `False` | Enable or disable the video overlay. | +| `--disable-selftext` | `RYBO_DISABLE_SELFTEXT` | `disable_selftext` | `False` | Enable or disable self-text video generation. | +| `--enable-background` | `RYBO_ENABLE_BACKGROUND` | `enable_background` | `False` | Enable or disable adding a background to the video. | +| `--enable-mentions` | `RYBO_ENABLE_MENTIONS` | `enable_mentions` | `False` | Check the reddit account for user mentions. | +| `--enable-nsfw` | `RYBO_ENABLE_NSFW` | `enable_nsfw` | `False` | Include, or ignore posts tagged as Not Safe for Work. | +| `--enable-upload` | `RYBO_ENABLE_UPLOAD` | `enable_upload` | `False` | Enable or disable uploading videos to YouTube. | +| `--orientation` | `RYBO_ORIENTATION` | `orientation` | `landscape` | Set the video orientation. Choices are **portrait** or **Landscape**. | +| `--shorts` | `RYBO_SHORTS` | `shorts` | `False` | Enable or disable generating a YouTube shorts video. | +| `--sort` | `RYBO_SORT` | `sort` | `hot` | Set the sorting order when scanning Reddit posts. Choices are **top** or **hot**. | +| `--submission-score` | `RYBO_SUBMISSION_SCORE` | `submission_score` | `5000` | Minimum submission score threshold. | +| `--subreddits` | `RYBO_SUBREDDITS` | `subreddits` | | List of subreddits to scan, where each subreddit is separated with a **+**. | +| `--story-mode` | `RYBO_STORY_MODE` | `story_mode` | `False` | Enable or disable video generation for the post title and selftext only, disables user comments. | +| `--thumbnail-only` | `RYBO_THUMBNAIL_ONLY` | `thumbnail_only` | `False` | Enable or disable generation of just the video thumbnails. | +| `--time` | `RYBO_TIME` | `time` | `day` | Filter Reddit submissions by time. Choices are **all**, **day**, **hour**, **month**, **week** or **year**. | +| `--total-posts` | `RYBO_TOTAL_POSTS` | `total_posts` | `10` | Total number of reddit submissions to process. | +| `--url` | `RYBO_URL` | `url` | | Generate a video for a single Reddit submission. | +| `--video-length` | `RYBO_VIDEO_LENGTH` | `video_length` | `600` | Sets how long the generated video will be, in seconds. | +| `--voice-engine` | `RYBO_VOICE_ENGINE` | `voice_engine` | `edge-tts` | Specify which text-to-speech engine should be used to narrate the video. Choices are **polly**, **balcon**, **gtts**, **tiktok**, **edge-tts**, **streamlabspolly**. | + +
+ +### Downloading video backgrounds using yt-dlp If you want to add a video background then install yt-dlp : @@ -114,75 +222,74 @@ cd assets/backgrounds yt-dlp --playlist-items 1:10 -f 22 --output "%(uploader)s_%(id)s.%(ext)s" https://www.youtube.com/playlist?list=PLGmxyVGSCDKvmLInHxJ9VdiwEb82Lxd2E ``` -# Help +### Help You can view available parameters by passing in `--help` : -``` -python app.py --help +```powershell +rybo --help + +============================== YOUTUBE REDDIT BOT =============================== +OS Version : Linux 5.15.90.1-microsoft-standard-WSL2 +Python Version : 3.11.3 (main, May 4 2023, 05:53:32) [GCC 10.2.1 20210110] +Rybo Version : 0.0.1 + +usage: rybo [-h] [--config CONFIG] [--version] [--background-directory] [-c {text,reddit}] [-o] [--disable-selftext] [-b] [--enable-mentions] [-n] [-p] + [--orientation {landscape,portrait}] [--shorts] [--sort {top,hot}] [--submission-score SUBMISSION_SCORE] [--subreddits SUBREDDITS] [-s] [-t] + [--time {all,day,hour,month,week,year}] [--total-posts TOTAL_POSTS] [-u URL] [-l VIDEO_LENGTH] + [--voice-engine {polly,balcon,gtts,tiktok,edge-tts,streamlabspolly}] -##### YOUTUBE REDDIT BOT ##### -usage: app.py [-h] [-l VIDEO_LENGTH] [-o] [-s] [-t] [-u URL] +Generate vidoes from reddit posts. options: -h, --help show this help message and exit - -l VIDEO_LENGTH, --video-length VIDEO_LENGTH - Set how long you want the video to be + --config CONFIG Path to the configuration file. + --version show programs version number and exit + --background-directory + Folder path to video backgrounds. + -c {text,reddit}, --comment-style {text,reddit} + Specify text based or reddit image comments. -o, --disable-overlay - Disable video overlay - -s, --story-mode Generate video for post title and selftext only, disables user comments - -t, --thumbnail-only Generate thumbnail image only + Disable video overlay. + --disable-selftext Disable selftext video generation. + -b, --enable-background + Enable video backgrounds. + --enable-mentions Check reddit account for u mentions. + -n, --enable-nsfw Allow NSFW Content. + -p, --enable-upload Upload video to youtube,requires client_secret.json and credentials. storage to be valid. + --orientation {landscape,portrait} + Sort Reddit posts by. + --shorts Generate Youtube Shorts Video. + --sort {top,hot} Sort Reddit posts by. + --submission-score SUBMISSION_SCORE + Minimum submission score threshold. + --subreddits SUBREDDITS + Specify Subreddits, seperate with +. + -s, --story-mode Generate video for post title and selftext only, disables user comments. + -t, --thumbnail-only Generate thumbnail image only. + --time {all,day,hour,month,week,year} + Filter by time. + --total-posts TOTAL_POSTS + Enable video backgrounds. -u URL, --url URL Specify Reddit post url, seperate with a comma for multiple posts. + -l VIDEO_LENGTH, --video-length VIDEO_LENGTH + Set how long you want the video to be. + --voice-engine {polly,balcon,gtts,tiktok,edge-tts,streamlabspolly} + Specify which text to speech engine to use. ``` -## Generate a Video for a Specific Post - -or if you want to generate a video for a specific reddit post you can specify it via the `--url` param : - -``` -python app.py --url https://www.reddit.com/r/AskReddit/comments/hvsxty/which_legendary_reddit_post_comment_can_you_still/ -``` - -or you can do multiple url's by seperating with a comma, ie : - -``` -python app.py --url https://www.reddit.com/r/post1,https://www.reddit.com/r/post2,https://www.reddit.com/r/post3 -``` - -## Generate Only Thumbnails - -if you want to generate only thumbnails you can specify `--thumbnail-only` mode, this will skip video compilation process : - -``` -python app.py --thumbnail-only -``` - -## Enable a Newscaster - -If you want to enable a Newscaster, edit settings.py and set : - -``` -enable_newscaster = True -``` - -![](assets/newscaster.png) - -If the newcaster video has a green screen you can remove it with the following settings, -use an eye dropper to get the RGB colour of the greenscreen and set it to have it removed : +## Customising -``` -newscaster_remove_greenscreen = True -newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour -newscaster_greenscreen_remove_threshold = 100 -``` +Theres quite a few options you can customise in the `settings.py` file. -## Settings.py File +These will at some point be moved into the main `rybo.yaml` configuration file. -Theres quite a few options you can customise in the `settings.py` file : +
+Click to view all available customisation options -Specify which subreddits you want to scrape : +### Specify Subreddits to Scrape -``` +```python subreddits = [ "AmItheAsshole", "antiwork", @@ -200,21 +307,23 @@ subreddits = [ ] ``` -Subreddits to exclude : +### Exclude Subreddits -``` +```python subreddits_excluded = [ "r/CFB", ] ``` -Filter out reddit posts via specified keywords +### Filter Reddit submissions by keyword -``` +```python banned_keywords =["my", "nasty", "keywords"] ``` -Change the Text to Speech engine you want to use, note AWS Polly requires and AWS account and auth tokens and can incur costs : +### Change the Text to Speech Engine + +note AWS Polly requires and AWS account and auth tokens and can incur costs : Supports Speech Engines : @@ -222,71 +331,86 @@ Supports Speech Engines : * [Balcon](http://www.cross-plus-a.com/bconsole.htm) * Python [gtts](https://gtts.readthedocs.io/en/latest/) -``` +```python # choices "polly","balcon","gtts" voice_engine = "polly" ``` -Total number of reddit Videos to generate +### Limit the number of generated videos -``` +```python total_posts_to_process = 5 ``` The next settings are to automatically filter out posts +### Skip Reddit posts by submission score + Skip reddit posts that less than this amount of updates -``` +```python minimum_submission_score = 5000 ``` +### Filter Reddit posts by title length + Filtering out reddit posts based on the reddit post title length -``` +```python title_length_minimum = 20 title_length_maximum = 100 ``` +### Filter Reddit posts by self text length + Filter out posts that exceed the maximum self text length -``` +```python maximum_length_self_text = 5000 ``` +### Filter Reddit posts by comment count + Filter out reddit posts that don't have enough comments -``` +```python minimum_num_comments = 200 ``` +### Limit number of posts to process + Only attempt to process a maximum amount of reddit posts -``` +```python submission_limit = 1000 ``` +### Configure number of thumbnails to generate + Specify how many thumbnail images you want to generate -``` +```python number_of_thumbnails = 3 ``` +### Set the maximum video length Specify the maximum video length -``` +```python max_video_length = 600 # Seconds ``` +### Set maximum number of comments to include + Specify maximum amount of comments to generate in the video -``` +```python comment_limit = 600 ``` -Specifying various folder paths +### Specify folder paths -``` +```python assets_directory = "assets" temp_directory = "temp" audio_directory = str(Path("temp")) @@ -299,83 +423,91 @@ video_overlay_filepath = str(Path(assets_directory,"particles.mp4")) videos_directory = "videos" ``` -Specify video height and width +### Set video dimensions -``` +```python video_height = 720 video_width = 1280 clip_size = (video_width, video_height) ``` -Skip compiling the video and just exit instead +### Skip video compilation -``` +Skip compiling the video and just exit instead. + +```python enable_compilation = True ``` -Skip uploading to youtube +### Skip YouTube uploading -``` +```python enable_upload = False ``` +### Add a video overlay + Add a video overlay to the video, for example snow falling effect -``` +```python enable_overlay = True ``` +### Add a Newscaster + Add in a newscaster reader to the video -``` +```python enable_newscaster = True ``` -If newcaster video is a green screen attempt to remove the green screen +If the newcaster video is a green screen, attempt to remove the green screen -``` +```python newscaster_remove_greenscreen = True ``` Specify the color of the green screen in RGB -``` +```python newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour ``` -The higher the greenscreen threshold number the more it will attempt to remove +The higher the greenscreen threshold number the more it will attempt to remove. -``` +```python newscaster_greenscreen_remove_threshold = 100 ``` Path to newcaster file -``` +```python newscaster_filepath = str(Path(assets_directory,"newscaster.mp4").resolve()) ``` Position on the screen of the newscaster -``` +```python newscaster_position = ("left","bottom") ``` The size of the newscaster -``` +```python newcaster_size = (video_width * 0.5, video_height * 0.5) ``` +### Add a pause after each TTS file + Add a pause after each text to speech audio file -``` +```python pause = 1 # Pause after speech ``` -Text style settings +### Modify appearnace of text -``` +```python text_bg_color = "#1A1A1B" text_bg_opacity = 1 text_color = "white" @@ -383,8 +515,54 @@ text_font = "Verdana-Bold" text_fontsize = 32 ``` +### Download images from Lexica + Download images from lexica or skip trying to download -``` +```python lexica_download_enabled = True ``` +
+ +## Tips and Tricks + +### Generate a Video for a Specific Post + +or if you want to generate a video for a specific reddit post you can specify it via the `--url` param : + +```powershell +rybo --url https://www.reddit.com/r/AskReddit/comments/hvsxty/which_legendary_reddit_post_comment_can_you_still/ +``` + +or you can do multiple url's by seperating with a comma, ie : + +```powershell +rybo --url https://www.reddit.com/r/post1,https://www.reddit.com/r/post2,https://www.reddit.com/r/post3 +``` + +### Generate Only Thumbnails + +if you want to generate only thumbnails you can specify `--thumbnail-only` mode, this will skip video compilation process : + +```powershell +rybo --thumbnail-only +``` + +### Enable a Newscaster + +If you want to enable a Newscaster, edit settings.py and set : + +```python +enable_newscaster = True +``` + +![](assets/newscaster.png) + +If the newcaster video has a green screen you can remove it with the following settings, +use an eye dropper to get the RGB colour of the greenscreen and set it to have it removed : + +```python +newscaster_remove_greenscreen = True +newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour +newscaster_greenscreen_remove_threshold = 100 +``` \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index ce19491..0000000 --- a/app.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Main entrypoint for the bot.""" -import logging -import os -import platform -import sys -from argparse import ArgumentParser, Namespace -from pathlib import Path -from typing import List - -from praw.models import Submission - -import config.settings as settings -import reddit.reddit as reddit -import thumbnail.thumbnail as thumbnail -import video_generation.video as vid -from csvmgr import CsvWriter -from utils.common import create_directory, safe_filename - -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) - - -class Video: - """Metadata used to splice content together to form a video.""" - - def __init__(self, submission): - """Initialize a Video instance. - - Args: - submission: The Reddit post to be converted into a video. - """ - self.submission = submission - self.comments = [] - self.clips = [] - self.background = None - self.music = None - self.thumbnail_path = None - self.folder_path = None - self.video_filepath = None - - -def process_submissions(submissions: List[Submission]) -> None: - """Prepare multiple reddit posts for conversion into YouTube videos. - - Args: - submissions: A list of zero or more Reddit posts to be converted. - """ - post_total: int = settings.total_posts_to_process - post_count: int = 0 - - for submission in submissions: - title_path: str = safe_filename(submission.title) - folder_path: Path = Path( - settings.videos_directory, f"{submission.id}_{title_path}" - ) - video_filepath: Path = Path(folder_path, "final.mp4") - if video_filepath.exists() or csvwriter.is_uploaded(submission.id): - print(f"Final video already processed : {submission.id}") - else: - process_submission(submission) - post_count += 1 - if post_count >= post_total: - print("Reached post count total!") - break - - -def process_submission(submission: Submission) -> None: - """Prepare a reddit post for conversion into a YouTube video. - - Args: - submission: The Reddit post to be converted into to a video. - """ - print("===== PROCESSING SUBMISSION =====") - print( - f"{str(submission.id)}, {str(submission.score)}, \ - {str(submission.num_comments)}, \ - {len(submission.selftext)}, \ - {submission.subreddit_name_prefixed}, \ - {submission.title}" - ) - video: Video = Video(submission) - title_path: str = safe_filename(submission.title) - - # Create Video Directories - video.folder_path: str = str( - Path(settings.videos_directory, f"{submission.id}_{title_path}") - ) - - create_directory(video.folder_path) - - video.video_filepath = str(Path(video.folder_path, "final.mp4")) - - if os.path.exists(video.video_filepath): - print(f"Final video already compiled : {video.video_filepath}") - else: - # Generate Thumbnail - - thumbnails: List[Path] = thumbnail.generate( - video_directory=video.folder_path, - subreddit=submission.subreddit_name_prefixed, - title=submission.title, - number_of_thumbnails=settings.number_of_thumbnails, - ) - - if thumbnails: - video.thumbnail_path = thumbnails[0] - - if args.thumbnail_only: - print("Generating Thumbnail only skipping video compile!") - else: - vid.create( - video_directory=video.folder_path, - post=submission, - thumbnails=thumbnails, - ) - - -def banner(): - """Display the CLIs banner.""" - print("##### YOUTUBE REDDIT BOT #####") - - -def print_version_info(): - """Display basic environment information.""" - print(f"OS Version : {platform.system()} {platform.release()}") - print(f"Python Version : {sys.version}") - - -def get_args() -> Namespace: - """Generate arguments supported by the CLI utility. - - Returns: - An argparse Namepsace containing the supported CLI parameters. - """ - parser: ArgumentParser = ArgumentParser() - - parser.add_argument( - "--enable-mentions", - action="store_true", - help="Check reddit account for u mentions", - ) - - parser.add_argument( - "--disable-selftext", - action="store_true", - help="Disable selftext video generation", - ) - - parser.add_argument( - "--voice-engine", - help="Specify which text to speech engine to use", - choices=["polly", "balcon", "gtts", "tiktok", "edge-tts", "streamlabspolly"], - ) - - parser.add_argument( - "-c", - "--comment-style", - help="Specify text based or reddit image comments", - choices=["text", "reddit"], - ) - - parser.add_argument( - "-l", "--video-length", help="Set how long you want the video to be", type=int - ) - - parser.add_argument( - "-n", "--enable-nsfw", action="store_true", help="Allow NSFW Content" - ) - - parser.add_argument( - "-o", "--disable-overlay", action="store_true", help="Disable video overlay" - ) - - parser.add_argument( - "-s", - "--story-mode", - action="store_true", - help="Generate video for post title and selftext only,\ - disables user comments", - ) - - parser.add_argument( - "-t", - "--thumbnail-only", - action="store_true", - help="Generate thumbnail image only", - ) - - parser.add_argument( - "-p", - "--enable-upload", - action="store_true", - help="Upload video to youtube, \ - requires client_secret.json and \ - credentials.storage to be valid", - ) - - parser.add_argument( - "-u", - "--url", - help="Specify Reddit post url, \ - seperate with a comma for multiple posts.", - ) - - parser.add_argument("--subreddits", help="Specify Subreddits, seperate with +") - - parser.add_argument( - "-b", - "--enable-background", - action="store_true", - help="Enable video backgrounds", - ) - - parser.add_argument("--total-posts", type=int, help="Enable video backgrounds") - - parser.add_argument( - "--submission-score", type=int, help="Minimum submission score threshold" - ) - - parser.add_argument( - "--background-directory", help="Folder path to video backgrounds" - ) - - parser.add_argument("--sort", choices=["top", "hot"], help="Sort Reddit posts by") - - parser.add_argument( - "--time", - choices=["all", "day", "hour", "month", "week", "year"], - default="day", - help="Filter by time", - ) - - parser.add_argument( - "--orientation", - choices=["landscape", "portrait"], - default="landscape", - help="Sort Reddit posts by", - ) - - parser.add_argument( - "--shorts", action="store_true", help="Generate Youtube Shorts Video" - ) - - args = parser.parse_args() - - if args.orientation: - settings.orientation = args.orientation - if args.orientation == "portrait": - settings.video_height = settings.vertical_video_height - settings.video_width = settings.vertical_video_width - logging.info("Setting Orientation to : %s", settings.orientation) - logging.info("Setting video_height to : %s", settings.video_height) - logging.info("Setting video_width to : %s", settings.video_width) - - if args.shorts: - logging.info("Generating Youtube Shorts Video") - settings.orientation = "portrait" - settings.video_height = settings.vertical_video_height - settings.video_width = settings.vertical_video_width - settings.max_video_length = 59 - settings.add_hashtag_shorts_to_description = True - - if args.enable_mentions: - settings.enable_reddit_mentions = True - logging.info("Enable Generate Videos from User Mentions") - - if args.submission_score: - settings.minimum_submission_score = args.submission_score - logging.info( - "Setting Reddit Post Minimum Submission Score : %s", - settings.minimum_submission_score, - ) - - if args.sort: - settings.reddit_post_sort = args.sort - logging.info("Setting Reddit Post Sort : %s", settings.reddit_post_sort) - if args.time: - settings.reddit_post_time_filter = args.time - logging.info( - "Setting Reddit Post Time Filter : %s", settings.reddit_post_time_filter - ) - - if args.background_directory: - logging.info( - "Setting video background directory : %s", args.background_directory - ) - settings.background_directory = args.background_directory - - if args.total_posts: - logging.info("Total Posts to process : %s", args.total_posts) - settings.total_posts_to_process = args.total_posts - - if args.comment_style: - logging.info("Setting comment style to : %s", args.comment_style) - settings.commentstyle = args.comment_style - - if args.voice_engine: - logging.info("Setting speech engine to : %s", args.voice_engine) - settings.voice_engine = args.voice_engine - - if args.video_length: - logging.info("Setting video length to : %s seconds", args.video_length) - settings.max_video_length = args.video_length - - if args.disable_overlay: - logging.info("Disabling Video Overlay") - settings.enable_overlay = False - - if args.enable_nsfw: - logging.info("Enable NSFW Content") - settings.enable_nsfw_content = True - - if args.story_mode: - logging.info("Story Mode Enabled!") - settings.enable_comments = False - - if args.disable_selftext: - logging.info("Disabled SelfText!") - settings.enable_selftext = False - - if args.enable_upload: - logging.info("Upload video enabled!") - settings.enable_upload = True - - if args.subreddits: - logging.info("Subreddits :") - settings.subreddits = args.subreddits.split("+") - print(settings.subreddits) - - if args.enable_background: - logging.info("Enabling Video Background!") - settings.enable_background = True - - return args - - -if __name__ == "__main__": - banner() - print_version_info() - args: Namespace = get_args() - csvwriter: CsvWriter = CsvWriter() - csvwriter.initialise_csv() - - submissions: List[Submission] = [] - - if args.url: - urls = args.url.split(",") - for url in urls: - submissions.append(reddit.get_reddit_submission(url)) - else: - if settings.enable_reddit_mentions: - logging.info("Getting Reddit Mentions") - mention_posts = reddit.get_reddit_mentions() - for mention_post in mention_posts: - logging.info("Reddit Mention : %s", mention_post) - submissions.append(reddit.get_reddit_submission(mention_post)) - - reddit_posts: List[Submission] = reddit.posts() - for reddit_post in reddit_posts: - submissions.append(reddit_post) - - submissions = reddit.get_valid_submissions(submissions) - - if submissions: - process_submissions(submissions) diff --git a/config/auth-env.py b/config/auth-env.py deleted file mode 100644 index 9d8499a..0000000 --- a/config/auth-env.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Load authentication credentials from environment variables.""" -import os - -print("Getting Secrets from ENV's") -# Reddit Praw -praw_client_id = os.environ.get("PRAW_CLIENT_ID") -praw_client_secret = os.environ.get("PRAW_CLIENT_SECRET") -praw_user_agent = os.environ.get("PRAW_USER_AGENT") -praw_password = os.environ.get("PRAW_PASSWORD") -praw_username = os.environ.get("PRAW_USERNAME") - -# Amazon Polly -aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") -aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - -# Rumble -rumble_username = os.environ.get("RUMBLE_USERNAME") -rumble_password = os.environ.get("RUMBLE_PASSWORD") diff --git a/config/auth-example.py b/config/auth-example.py deleted file mode 100644 index cfe7fe7..0000000 --- a/config/auth-example.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Authentication credentials used by the bot.""" - -# FIXME: move credentials out of file. - -# Reddit Praw -praw_client_id = "xxxxxx" # noqa: S105 -praw_client_secret = "xxxxxx" # noqa: S105 -praw_user_agent = "xxxxxx" -praw_username = "xxxxxx" -praw_password = "xxxxxx" # noqa: S105 - -# Amazon Polly -aws_access_key_id = "xxxxxx" -aws_secret_access_key = "xxxxxx" # noqa: S105 - -# Rumble -rumble_username = "xxxxxx" -rumble_password = "xxxxxx" # noqa: S105 diff --git a/comments/cookie-dark-mode.json b/cookies/cookie-dark-mode.json similarity index 100% rename from comments/cookie-dark-mode.json rename to cookies/cookie-dark-mode.json diff --git a/comments/cookie-light-mode.json b/cookies/cookie-light-mode.json similarity index 100% rename from comments/cookie-light-mode.json rename to cookies/cookie-light-mode.json diff --git a/requirements-dev.txt b/requirements-dev.txt index 12ccdc6..cb8fe6d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8-bandit==4.1.1 flake8-black==0.3.6 flake8-bugbear==23.3.23 flake8-docstrings==1.7.0 -flake8-isort==4.0.0 +#flake8-isort==6.0.0 flake8-pyproject==1.2.3 flake8-secure-coding-standard==1.4.0 mypy==1.2.0 diff --git a/requirements.txt b/requirements.txt index dc3d538..6ee39c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ boto3==1.26.123 bs4==0.0.1 +colorama==0.4.6 +configparser==5.3.0 edge-tts==6.1.3 emoji==2.2.0 gtts==2.3.2 @@ -11,6 +13,7 @@ praw==7.7.0 pytime==0.2.3 pyyaml==6.0.0 rich==13.3.5 +ruamel.yaml==0.17.26 selenium==4.9.0 simple-youtube-api==0.2.8 varname==0.11.1 diff --git a/rybo.yaml b/rybo.yaml new file mode 100644 index 0000000..0a06954 --- /dev/null +++ b/rybo.yaml @@ -0,0 +1,82 @@ +--- + +# Folder path to videos that will be used for the video background. +background_directory: assets/backgrounds + +# Use text based, or image based Reddit comments. Choices are **text** or **image**. +comment_style: reddit + +# Enable or disable the video overlay. +disable_overlay: False + +# Enable or disable self-text video generation. +disable_selftext: False + +# Enable or disable adding a background to the video. +enable_background: False + +# Check the reddit account for user mentions. +enable_mentions: False + +# Include, or ignore posts tagged as Not Safe for Work. +enable_nsfw: False + +# Enable or disable uploading videos to YouTube. +enable_upload: False + +# Set the video orientation. Choices are "portrait" or "Landscape". +orientation: landscape + +# Polly credentials for the AWS API. +polly: + access_key_id: + secret_access_key: + +# Reddit credentials for the API and Web UI. +reddit: + client_id: + client_secret: + username: + password: + +# Rumble credentials for the Web UI. +rumble: + username: + password: + +# Enable or disable generating a YouTube shorts video. +shorts: False + +# Set the sorting order when scanning Reddit posts. Choices are "top" or "hot". +sort: hot + +# Enable or disable video generation for the post title and selftext only, +# disables user comments. +story_mode: False + +# Minimum submission score threshold. +submission_score: 5000 + +# List of subreddits to scan, where each subreddit is separated with a "+". +subreddits: antiwork+AskMen+askreddit+ChoosingBeggars+confession+confessions+hatemyjob+NoStupidQuestions+pettyrevenge+Showerthoughts+TooAfraidToAsk+TwoXChromosomes+unpopularopinion + +# Enable or disable generation of just the video thumbnails. +thumbnail_only: False + +# Filter Reddit submissions by time. Choices are "all", "day", "hour", +# "month", "week" or "year". +time: day + +# Total number of reddit submissions to process. +total_posts: 10 + +# Generate a video for a single Reddit submission. +url: + +# Sets how long the generated video will be, in seconds. +video_length: 600 + +# Specify which text-to-speech engine should be used to narrate the video. +# Choices are "polly", "balcon", "gtts", "tiktok", "edge-tts" or +# "streamlabspolly". +voice_engine: edge-tts diff --git a/src/rybo/cli/__init__.py b/src/rybo/cli/__init__.py new file mode 100644 index 0000000..1184d0d --- /dev/null +++ b/src/rybo/cli/__init__.py @@ -0,0 +1 @@ +"""Rybo command line utility.""" diff --git a/src/rybo/cli/rybo.py b/src/rybo/cli/rybo.py new file mode 100644 index 0000000..bf8ce9f --- /dev/null +++ b/src/rybo/cli/rybo.py @@ -0,0 +1,525 @@ +"""Command line utility to generate YouTube videos from Reddit posts.""" + +import logging +import logging.config +import os +import platform +import sys +from argparse import ArgumentParser, Namespace +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List + +import colorama +from colorama import Fore +from ruamel.yaml import YAML + +from rybo import __version__ +from rybo.commands import RyboCommand +from rybo.config import settings +from rybo.logging import DEFAULT_LOG_CONFIG +from rybo.utils import EnvDefault, EnvFlagDefault + +# TODO: Placeholder for sub-commands. +# from rybo.commands.thumbnails import CreateThumbnailCommand +# from rybo.commands.video import CreateVideoCommand +# from rybo.commands.reddit import ExtractRedditCommand +# from rbo.commands.youtube import PublishYouTubeCommand + + +DEFAULT_CONFIG_FILE: Path = Path.joinpath(Path.home(), "rybo.yaml") +logger = logging.getLogger(__name__) + + +def cli() -> None: + """Create the rybo command line utility. + + Configuration options are loaded in the following order: + + 1. From a configuration file (default: $HOME/rybo.yaml). + 2. From environment variables that start with the prefix `RYBO_`. + 3. From user provided CLI parameters. + + Returns: + Returns an ArgumentParser that contains all of the sub-commands by rybo. + """ + _display_banner() + + # TODO: This needs refactoring + config_argparse: ArgumentParser = _configfile_parser() + config_args, _ = config_argparse.parse_known_args() + + defaults: Dict[str, Any] = DEFAULT_LOG_CONFIG + + if config_args.config is None: + config_args.config = DEFAULT_CONFIG_FILE + + cfg: Dict[str, Any] = _load_config(config_args) + cfg |= defaults + + # There are some additional environment variables that can be set to + # override configuration provided as CLI parameters, or via a configuration + # file, so need to load these last and write over the top of any values + # that already exist in the defaults dictionary. + envvars: Dict[str, Any] = _parse_env_overrides() + cfg |= envvars + + # Allows the user to change logging behaviour via a configuration file. + logging.config.dictConfig(cfg) + + parser: ArgumentParser = _cli(config_argparse, cfg) + _add_standard_args(parser) + + # TODO: placeholder for sub-commands + # subparser = parser.add_subparsers() + # _add_create_command(subparser) + + args: Namespace = parser.parse_args() + _log_configuration(args) + + print(args) + + command: Any = args.cmd + command.execute(args) + + +def _add_standard_args(parser: ArgumentParser) -> None: + """Common CLI command arguments. + + Args: + parser: Main parser that provides the rybo utility. + """ # noqa: D401 + # TODO: Needs refactoring + parser.add_argument( + "--version", + action="version", + version="%(prog)s {version}".format(version=__version__), + ) + parser.add_argument( + "--background-directory", + action=EnvFlagDefault, + envvar="RYBO_BACKGROUND_DIRECTORY", + help="Folder path to video backgrounds (env var: RYBO_BACKGROUND_DIRECORY).", + ) + parser.add_argument( + "-c", + "--comment-style", + action=EnvDefault, + envvar="RYBO_COMMENT_STYLE", + help="Specify text based or reddit image comments \ + (env var: RYBO_COMMENT_STYLE).", + choices=["text", "reddit"], + ) + parser.add_argument( + "-o", + "--disable-overlay", + action=EnvFlagDefault, + envvar="RYBO_DISABLE_OVERLAY", + help="Disable video overlay (env var: RYBO_DISABLE_OVERLAY).", + ) + parser.add_argument( + "--disable-selftext", + action=EnvFlagDefault, + envvar="RYBO_DISABLE_SELFTEXT", + help="Disable selftext video generation (env var: RYBO_DISABLE_SELFTEXT).", + ) + parser.add_argument( + "-b", + "--enable-background", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_BACKGROUND", + help="Enable video backgrounds (env var: RYBO_ENABLE_BACKGROUND).", + ) + parser.add_argument( + "--enable-mentions", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_MENTIONS", + help="Check reddit account for user mentions (env var: RYBO_ENABLE_MENTIONS).", + ) + parser.add_argument( + "-n", + "--enable-nsfw", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_NSFW", + help="Allow NSFW Content (env var: RYBO_ENABLE_NSFW).", + ) + parser.add_argument( + "-p", + "--enable-upload", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_UPLOAD", + help="Upload video to youtube,requires client_secret.json and credentials.\ + storage to be valid (env var: RYBO_ENABLE_UPLOAD).", + ) + parser.add_argument( + "--orientation", + choices=["landscape", "portrait"], + action=EnvDefault, + envvar="RYBO_ORIENTATION", + default="landscape", + help="Sort Reddit posts by (env var: RYBO_ORIENTATION).", + ) + parser.add_argument( + "--shorts", + action=EnvFlagDefault, + envvar="RYBO_SHORTS", + help="Generate Youtube Shorts Video (env var: RYBO_SHORTS).", + ) + parser.add_argument( + "--sort", + action=EnvDefault, + envvar="RYBO_SORT", + choices=["top", "hot"], + help="Sort Reddit posts by (env var: RYBO_SORT).", + ) + parser.add_argument( + "--submission-score", + action=EnvDefault, + envvar="RYBO_SUBMISSION_SCORE", + type=int, + help="Minimum submission score threshold (env var: RYBO_SUBMISSION_SCORE).", + ) + parser.add_argument( + "--subreddits", + action=EnvDefault, + envvar="RYBO_SUBREDDITS", + help="Specify Subreddits, seperate with + (env var: RYBO_SUBREDDITS).", + ) + parser.add_argument( + "-s", + "--story-mode", + action=EnvFlagDefault, + envvar="RYBO_STORY_MODE", + help="Generate video for post title and selftext only, disables user comments \ + (env var: RYBO_STORY_MODE).", + ) + parser.add_argument( + "-t", + "--thumbnail-only", + action=EnvFlagDefault, + envvar="RYBO_THUMBNAIL_ONLY", + help="Generate thumbnail image only (env var: RYBO_THUMBNAIL_ONLY).", + ) + parser.add_argument( + "--time", + action=EnvDefault, + envvar="RYBO_TIME", + choices=["all", "day", "hour", "month", "week", "year"], + default="day", + help="Filter by time (env var: RYBO_TIME).", + ) + parser.add_argument( + "--total-posts", + action=EnvDefault, + envvar="RYBO_TOTAL_POSTS", + type=int, + help="Number of posts to process (env var: RYBO_TOTAL_POSTS).", + ) + parser.add_argument( + "-u", + "--url", + action=EnvDefault, + envvar="RYBO_URL", + help="Specify Reddit post url, seperate with a comma for multiple posts \ + (env var: RYBO_URL).", + ) + parser.add_argument( + "-l", + "--video-length", + action=EnvDefault, + envvar="RYBO_VIDEO_LENGTH", + type=int, + help="Set how long you want the video to be (env var: RYBO_VIDEO_LENGTH).", + ) + parser.add_argument( + "--voice-engine", + action=EnvDefault, + envvar="RYBO_VOICE_ENGINE", + choices=["polly", "balcon", "gtts", "tiktok", "edge-tts", "streamlabspolly"], + help="Specify which text to speech engine to use (env var: RYBO_VOICE_ENGINE).", + ) + + args = parser.parse_args() + + if args.orientation: + settings.orientation = args.orientation + if args.orientation == "portrait": + settings.video_height = settings.vertical_video_height + settings.video_width = settings.vertical_video_width + + if args.shorts: + logger.info("Generating Youtube Shorts Video") + settings.orientation = "portrait" + settings.video_height = settings.vertical_video_height + settings.video_width = settings.vertical_video_width + settings.max_video_length = 59 + settings.add_hashtag_shorts_to_description = True + + if args.enable_mentions: + settings.enable_reddit_mentions = True + + if args.submission_score: + settings.minimum_submission_score = args.submission_score + + if args.sort: + settings.reddit_post_sort = args.sort + + if args.time: + settings.reddit_post_time_filter = args.time + + if args.background_directory: + settings.background_directory = args.background_directory + + if args.total_posts: + settings.total_posts_to_process = args.total_posts + + if args.comment_style: + settings.commentstyle = args.comment_style + + if args.voice_engine: + settings.voice_engine = args.voice_engine + + if args.video_length: + settings.max_video_length = args.video_length + + if args.disable_overlay: + settings.enable_overlay = False + + if args.enable_nsfw: + settings.enable_nsfw_content = True + + if args.story_mode: + settings.enable_comments = False + + if args.disable_selftext: + settings.enable_selftext = False + + if args.enable_upload: + settings.enable_upload = True + + if args.subreddits: + settings.subreddits = args.subreddits.split("+") + logger.info("Subreddits :") + logger.info(settings.subreddits) + + if args.enable_background: + settings.enable_background = True + + parser.set_defaults(cmd=RyboCommand(parser)) + + +def _cli(parser: ArgumentParser, defaults: Dict[str, Any]) -> ArgumentParser: + """Create the main CLI parser. + + Where a configuration file is found, this will load CLI options from the + file, over-riding any configuration set via environment variables. + + Args: + parser: Parser used to read the configuration file. + defaults: Dictionary that contains the CLI argument default values. + + Returns: + The main rybo parser. + """ + parsers: List[ArgumentParser] = [parser] + parser: ArgumentParser = ArgumentParser( + description="Generate vidoes from reddit posts.", parents=parsers + ) + parser.set_defaults(**defaults) + return parser + + +def _configfile_parser() -> ArgumentParser: + """Add the CLI argument used to load a custom config file. + + Returns: + An `ArgumentParser` instance with the nargument used to specify the + location of a custom configuration file. + """ + parser: ArgumentParser = ArgumentParser(prog=__file__, add_help=False) + parser.add_argument( + "--config", + default=DEFAULT_CONFIG_FILE, + action=EnvDefault, + envvar="RYBO_CONFIG_FILE", + help="Path to the configuration file.", + ) + return parser + + +def _display_banner() -> None: + """Display the CLIs banner.""" + colorama.init(autoreset=True) + + python_version: str = f"{'Python Version':<20} : {sys.version}" + os_version: str = f"{'OS Version':<20} : {platform.system()} {platform.release()}" + rybo_version: str = f"{'Rybo Version':<20} : {__version__}" + title: str = "YOUTUBE REDDIT BOT" + + python_version_width: int = len(python_version) + os_version_width: int = len(os_version) + + if len(python_version) > len(os_version): + title = f"{' YOUTUBE REDDIT BOT ':=^{python_version_width}}" + else: + title = f"{' YOUTUBE REDDIT BOT ':=^{os_version_width}}" + + print(f"{Fore.CYAN}{title}\n{os_version}\n{python_version}\n{rybo_version}\n") + + +def _load_config(args: Namespace) -> Dict[str, Any]: + """Load and parser a configuration file. + + Args: + args: CLI arguments. + + Returns: + A dictionary containing configuration options. + """ + config: Dict[str, Any] = {} + config_file: Path = Path(args.config) + if config_file.is_file(): + yaml: YAML = YAML(typ="safe") + + # ruamel doesn't have the same vulnerability as the standard yaml + # library, so it's safe to ignore SCS105 here. + config = yaml.load(config_file) # noqa: SCS105 + + return config + + +def _log_configuration(args: Namespace) -> None: + """Log the bot configuration settings. + + Args: + args: ArgumentParser containing the user provided runtime parameters. + """ + if args.orientation: + logger.info(f"{'Set orientation to':<45} : %s", settings.orientation) + logger.info(f"{'Set video height to':<45} : %s", settings.video_height) + logger.info(f"{'Set video width to':<45} : %s", settings.video_width) + + if args.shorts: + logger.info("Generating Youtube Shorts Video") + + if args.enable_mentions: + logger.info("Enable Generate Videos from User Mentions") + + if args.submission_score: + logger.info( + f"{'Setting Reddit Post Minimum Submission Score':<45} : %s", + settings.minimum_submission_score, + ) + + if args.sort: + logger.info(f"{'Setting Reddit Post Sort':<45} : %s", settings.reddit_post_sort) + + if args.time: + logger.info( + f"{'Setting Reddit Post Time Filter':<45} : %s", + settings.reddit_post_time_filter, + ) + + if args.background_directory: + logger.info( + f"{'Setting video background directory':<45} : %s", + args.background_directory, + ) + + if args.total_posts: + logger.info(f"{'Total Posts to process':<45} : %s", args.total_posts) + + if args.comment_style: + logger.info(f"{'Setting comment style to':<45} : %s", args.comment_style) + + if args.voice_engine: + logger.info(f"{'Setting speech engine to':<45} : %s", args.voice_engine) + + if args.video_length: + logger.info(f"{'Setting video length to':<45} : %s seconds", args.video_length) + + if args.disable_overlay: + logger.info("Disabling Video Overlay") + + if args.enable_nsfw: + logger.info("Enable NSFW Content") + + if args.story_mode: + logger.info("Story Mode Enabled!") + + if args.disable_selftext: + logger.info("Disabled SelfText!") + + if args.enable_upload: + logger.info("Upload video enabled!") + + if args.subreddits: + logger.info("Subreddits :") + settings.subreddits = args.subreddits.split("+") + logger.info(settings.subreddits) + + if args.enable_background: + logger.info("Enabling Video Background!") + + +def _parse_env_overrides() -> Dict[str, Any]: + """Load rybo configuration provided as environment variables. + + Returns: + A dictionary containing the configuration specified as + environment variables. + """ + cfg: Dict[str, Any] = defaultdict(dict) + + if os.environ.get("RYBO_REDDIT_CLIENT_ID"): + cfg["reddit"]["client_id"] = os.environ["RYBO_REDDIT_CLIENT_ID"] + + if os.environ.get("RYBO_REDDIT_CLIENT_SECRET"): + cfg["reddit"]["client_secret"] = os.environ["RYBO_REDDIT_CLIENT_SECRET"] + + if os.environ.get("RYBO_REDDIT_USERNAME"): + cfg["reddit"]["username"] = os.environ["RYBO_REDDIT_USERNAME"] + + if os.environ.get("RYBO_REDDIT_PASSWORD"): + cfg["reddit"]["password"] = os.environ["RYBO_REDDIT_PASSWORD"] + + if os.environ.get("RYBO_POLLY_ACCESS_KEY"): + cfg["polly"]["access_key_id"] = os.environ["RYBO_POLLY_ACCESS_KEY"] + + if os.environ.get("RYBO_POLLY_SECRET_ACCESS_KEY"): + cfg["polly"]["secret_access_key"] = os.environ["RYBO_POLLY_SECRET_ACCESS_KEY"] + + if os.environ.get("RYBO_RUMBLE_USERNAME"): + cfg["rumble"]["username"] = os.environ["RYBO_RUMBLE_USERNAME"] + + if os.environ.get("RYBO_RUMBLE_PASSWORD"): + cfg["rumble"]["password"] = os.environ["RYBO_RUMBLE_PASSWORD"] + + return dict(cfg) + + +# TODO: Placeholder for sub-commands. +# def _add_create_command(subparser: _SubParsersAction): +# """Sub-command used to create new Zephyr folders. + +# Args: +# subparser: Parent that the sub-command will belong to. +# """ +# parser = subparser.add_parser('create', help='Create a new folder.') +# parser.add_argument( +# '--project', +# required=True, +# help='Project key of the project that the folder will be created under.' +# ) +# parser.add_argument( +# '--name', +# required=False, +# help='Name of the folder.' +# ) +# parser.add_argument( +# '--type', +# required=False, +# choices=['plan', 'case', 'cycle'], +# help='Type of folder to create.', +# ) +# parser.set_defaults(cmd=CreateFolderCommand(parser)) diff --git a/src/rybo/commands/__init__.py b/src/rybo/commands/__init__.py new file mode 100644 index 0000000..a833557 --- /dev/null +++ b/src/rybo/commands/__init__.py @@ -0,0 +1,229 @@ +"""Commands that can be executed by the rybo CLI utility.""" + +import logging +from abc import ABC, abstractmethod +from argparse import ArgumentParser, Namespace +from pathlib import Path +from typing import List + +from praw.models import Submission + +import rybo.config.settings as settings +import rybo.thumbnail.thumbnail as thumbnail +import rybo.video_generation.video as vid +from rybo.reddit import reddit +from rybo.utils.common import create_directory, safe_filename +from rybo.utils.csvmgr import CsvWriter + +logger = logging.getLogger(__name__) + + +class Video: + """Metadata used to splice content together to form a video.""" + + def __init__(self, submission): + """Initialize a Video instance. + + Args: + submission: The Reddit post to be converted into a video. + """ + self.submission = submission + self.comments = [] + self.clips = [] + self.background = None + self.music = None + self.thumbnail_path = None + self.folder_path = None + self.video_filepath = None + + +class CommandBase(ABC): + """Base class for CLI commands.""" + + def __init__(self, cli: ArgumentParser) -> None: + """Initialise the sub-command. + + Args: + cli: ArgParse action that will execute this command. + """ + self._cli = cli + self._logger = logging.getLogger(__name__) + + @abstractmethod + def execute(self, args: Namespace) -> None: + """Execute the command. + + Args: + args: User provided CLI arguments. + """ + pass + + +class RyboCommand(CommandBase): + """Top level command for the rybo utility. + + _See Also_: + [CommandBase][zfr.commands.CommandBase] + """ + + def execute(self, args: Namespace) -> None: + """Execute the command. + + This will simply display help information, as the ```FolderCommand``` + command acts as a wrapper for sub-commands used to create/update + Zephyr folders. + + Args: + args: User provided CLI arguments. + """ + csvwriter: CsvWriter = CsvWriter() + csvwriter.initialise_csv() + + submissions: List[Submission] = [] + + if args.url: + urls: List[str] = args.url.split(",") + for url in urls: + submissions.append( + reddit.get_reddit_submission( + url=url, + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + ) + else: + if settings.enable_reddit_mentions: + logger.info("Getting Reddit Mentions") + mention_posts = reddit.get_reddit_mentions( + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + for mention_post in mention_posts: + logger.info("Reddit Mention : %s", mention_post) + submissions.append( + reddit.get_reddit_submission( + url=mention_post, + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + ) + + reddit_posts: List[Submission] = reddit.posts( + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + for reddit_post in reddit_posts: + submissions.append(reddit_post) + + submissions = reddit.get_valid_submissions(submissions) + + if submissions: + self._process_submissions( + submissions=submissions, + csvwriter=csvwriter, + thumbnail_only=args.thumbnail_only, + username=args.reddit["username"], + password=args.reddit["password"], + ) + + def _process_submissions( + self, + username: str, + password: str, + submissions: List[Submission], + csvwriter: CsvWriter, + thumbnail_only: bool = False, + ) -> None: + """Prepare multiple reddit posts for conversion into YouTube videos. + + Args: + submissions: A list of zero or more Reddit posts to be converted. + csvwriter: Helper object used to manage CSV files. + thumbnail_only: `True` to only generate a thumbnail and skip generating a + video, ottherwise `False` (default: False) + username: Reddit username. + password: Reddit password. + """ + post_total: int = settings.total_posts_to_process + post_count: int = 0 + + for submission in submissions: + title_path: str = safe_filename(submission.title) + folder_path: Path = Path( + settings.videos_directory, f"{submission.id}_{title_path}" + ) + video_filepath: Path = Path(folder_path, "final.mp4") + if video_filepath.exists() or csvwriter.is_uploaded(submission.id): + logger.info(f"Final video already processed : {submission.id}") + else: + self._process_submission( + submission=submission, + thumbnail_only=thumbnail_only, + username=username, + password=password, + ) + post_count += 1 + if post_count >= post_total: + logger.info("Reached post count total!") + break + + def _process_submission( + self, + username: str, + password: str, + submission: Submission, + thumbnail_only: bool = False, + ) -> None: + """Prepare a reddit post for conversion into a YouTube video. + + Args: + submission: The Reddit post to be converted into to a video. + thumbnail_only: `True` to only generate a thumbnail and skip generating a + video, ottherwise `False` (default: False) + username: Reddit username. + password: Reddit password. + """ + logger.info("===== PROCESSING SUBMISSION =====") + logger.info( + f"{str(submission.id)}, {str(submission.score)}, \ + {str(submission.num_comments)}, \ + {len(submission.selftext)}, \ + {submission.subreddit_name_prefixed}, \ + {submission.title}" + ) + video: Video = Video(submission) + title_path: str = safe_filename(submission.title) + + # Create Video Directories + video.folder_path: str = str( + Path(settings.videos_directory, f"{submission.id}_{title_path}") + ) + + create_directory(video.folder_path) + + video.video_filepath: Path = Path(video.folder_path, "final.mp4") + + if video.video_filepath.exists(): + logger.info(f"Final video already compiled : {video.video_filepath}") + else: + # Generate Thumbnail + thumbnails: List[Path] = thumbnail.generate( + video_directory=str(video.folder_path), + subreddit=submission.subreddit_name_prefixed, + title=submission.title, + number_of_thumbnails=settings.number_of_thumbnails, + ) + + if thumbnails: + video.thumbnail_path = thumbnails[0] + + if thumbnail_only: + logger.info("Generating Thumbnail only skipping video compile!") + else: + vid.create( + video_directory=video.folder_path, + post=submission, + thumbnails=thumbnails, + username=username, + password=password, + ) diff --git a/src/rybo/comments/__init__.py b/src/rybo/comments/__init__.py new file mode 100644 index 0000000..a314754 --- /dev/null +++ b/src/rybo/comments/__init__.py @@ -0,0 +1 @@ +"""Reddit comment processors.""" diff --git a/comments/screenshot.py b/src/rybo/comments/screenshot.py similarity index 81% rename from comments/screenshot.py rename to src/rybo/comments/screenshot.py index 6199c5c..c903e2a 100644 --- a/comments/screenshot.py +++ b/src/rybo/comments/screenshot.py @@ -1,5 +1,6 @@ """Take screenshots of Reddit comments.""" import json +import logging import os import re from io import TextIOWrapper @@ -10,27 +11,19 @@ from praw.models import Comment from rich.progress import track -import config.auth as auth -import config.settings as settings +from rybo.config import settings storymode = False - -def safe_filename(text: str): - """Replace spaces with an underscore. - - Args: - text: Filename to be sanitized. - - Returns: - A sanitized filename, where spaces have been replaced with underscores. - """ - text = text.replace(" ", "_") - return "".join([c for c in text if re.match(r"\w", c)])[:50] +logger = logging.getLogger(__name__) def download_screenshots_of_reddit_posts( - accepted_comments: List[Comment], url: str, video_directory: Path + accepted_comments: List[Comment], + url: str, + video_directory: Path, + username: str, + password: str, ) -> None: """Download screenshots of reddit posts as seen on the web. @@ -40,8 +33,10 @@ def download_screenshots_of_reddit_posts( accepted_comments: List of comments to be included in the video. url: URL of the Reddit content to be screenshotted. video_directory: Path where the screenshots will be saved. + username: Reddit username. + password: Reddit password. """ - print("Downloading screenshots of reddit posts...") + logger.info("Downloading screenshots of reddit posts...") # id = re.sub(r"[^\w\s-]", "", reddit_object.meta.id) # # ! Make sure the reddit screenshots folder exists # title_path = safe_filename(reddit_object.title) @@ -49,26 +44,26 @@ def download_screenshots_of_reddit_posts( # #Path(f"assets/temp/{id}/png").mkdir(parents=True, exist_ok=True) with sync_playwright() as p: - print("Launching Headless Browser...") + logger.debug("Launching Headless Browser...") browser = p.chromium.launch(headless=True) context = browser.new_context() context.set_default_timeout(settings.comment_screenshot_timeout) if settings.theme == "dark": cookie_file: TextIOWrapper = open( - f"{os.getcwd()}/comments/cookie-dark-mode.json", encoding="utf-8" + f"{os.getcwd()}/cookies/cookie-dark-mode.json", encoding="utf-8" ) else: cookie_file: TextIOWrapper = open( - f"{os.getcwd()}/comments/cookie-light-mode.json", encoding="utf-8" + f"{os.getcwd()}/cookies/cookie-light-mode.json", encoding="utf-8" ) # Get the thread screenshot page = context.new_page() page.goto("https://www.reddit.com/login") - page.type("#loginUsername", auth.praw_username) - page.type("#loginPassword", auth.praw_password) + page.type("#loginUsername", username) + page.type("#loginPassword", password) page.click('button[type="submit"]') page.wait_for_url("https://www.reddit.com/") @@ -81,7 +76,7 @@ def download_screenshots_of_reddit_posts( if page.locator('[data-testid="content-gate"]').is_visible(): # This means the post is NSFW and requires to click the proceed button. - print("Post is NSFW. You are spicy...") + logger.info("Post is NSFW. You are spicy...") page.locator('[data-testid="content-gate"] button').click() page.wait_for_load_state() # Wait for page to fully load @@ -101,7 +96,9 @@ def download_screenshots_of_reddit_posts( comment_path: Path = Path(f"{video_directory}/comment_{comment.id}.png") if comment_path.exists(): - print(f"Comment Screenshot already downloaded : {comment_path}") + logger.info( + f"Comment Screenshot already downloaded : {comment_path}" + ) else: if page.locator('[data-testid="content-gate"]').is_visible(): page.locator('[data-testid="content-gate"] button').click() @@ -113,7 +110,7 @@ def download_screenshots_of_reddit_posts( except Exception: # noqa: S110 pass - print("Screenshots downloaded Successfully.") + logger.info("Screenshots downloaded Successfully.") def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> None: @@ -125,10 +122,10 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> url: URL to take a screenshot of. video_directory: Path to save the screenshots to. """ - print("Downloading screenshots of reddit title...") - print(url) + logger.info("Downloading screenshots of reddit title...") + logger.info(url) with sync_playwright() as p: - print("Launching Headless Browser...") + logger.info("Launching Headless Browser...") browser = p.chromium.launch(headless=True) context = browser.new_context() @@ -149,7 +146,7 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> if page.locator('[data-testid="content-gate"]').is_visible(): # This means the post is NSFW and requires to click the proceed button. - print("Post is NSFW. You are spicy...") + logger.info("Post is NSFW. You are spicy...") page.locator('[data-testid="content-gate"] button').click() page.wait_for_load_state() # Wait for page to fully load @@ -166,4 +163,17 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> path=f"{video_directory}/title.png" ) - print("Title Screenshot downloaded Successfully.") + logger.info("Title Screenshot downloaded Successfully.") + + +def safe_filename(text: str) -> str: + """Replace spaces with an underscore. + + Args: + text: Filename to be sanitized. + + Returns: + A sanitized filename, where spaces have been replaced with underscores. + """ + text = text.replace(" ", "_") + return "".join([c for c in text if re.match(r"\w", c)])[:50] diff --git a/src/rybo/config/__init__.py b/src/rybo/config/__init__.py new file mode 100644 index 0000000..0eed9d6 --- /dev/null +++ b/src/rybo/config/__init__.py @@ -0,0 +1 @@ +"""Rybo configuration.""" diff --git a/config/settings.py b/src/rybo/config/settings.py similarity index 100% rename from config/settings.py rename to src/rybo/config/settings.py index 97d43cc..011f089 100644 --- a/config/settings.py +++ b/src/rybo/config/settings.py @@ -1,6 +1,6 @@ """Configuration settings for the bot.""" -from sys import platform from pathlib import Path +from sys import platform subreddits = [ "askreddit", diff --git a/src/rybo/logging.py b/src/rybo/logging.py new file mode 100644 index 0000000..bf5c8a5 --- /dev/null +++ b/src/rybo/logging.py @@ -0,0 +1,30 @@ +"""Default logging configuration, and associated utilities.""" + +from typing import Any, Dict + +DEFAULT_LOG_CONFIG: Dict[str, Any] = { + "version": 1, + "disable_existing_loggers": False, + "propogate": False, + "formatters": { + "fmt": { + "format": "%(asctime)s %(levelname)-8s %(message)s", + "datefmt": "%d-%m-%Y %H:%M:%S", + } + }, + "handlers": { + "fh": { + "class": "logging.handlers.RotatingFileHandler", + "filename": "rybo.log", + "maxBytes": 1048576, + "backupCount": 3, + "formatter": "fmt", + }, + "sh": { + "class": "logging.StreamHandler", + "formatter": "fmt", + "stream": "ext://sys.stdout", + }, + }, + "loggers": {"rybo": {"handlers": ["fh", "sh"], "level": "DEBUG"}}, +} diff --git a/src/rybo/publish/__init__.py b/src/rybo/publish/__init__.py new file mode 100644 index 0000000..22150c5 --- /dev/null +++ b/src/rybo/publish/__init__.py @@ -0,0 +1 @@ +"""Video publishers.""" diff --git a/publish/login.py b/src/rybo/publish/login.py similarity index 94% rename from publish/login.py rename to src/rybo/publish/login.py index 7fb9d43..0e4d663 100644 --- a/publish/login.py +++ b/src/rybo/publish/login.py @@ -1,5 +1,6 @@ """Login module.""" import json +import logging from typing import Dict, List from selenium.webdriver.common.by import By @@ -7,6 +8,28 @@ from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait +logger = logging.getLogger(__name__) + + +def confirm_logged_in(driver: WebDriver) -> bool: + """Confirm that the user is logged in. + + The browser needs to be navigated to a YouTube page. + + Args: + driver: Selenium webdrive. + + Returns: + `True` if the user is logged in, otherwise `False`. + """ + try: + WebDriverWait(driver, 5).until( + ec.element_to_be_clickable((By.ID, "avatar-btn")) + ) + return True + except TimeoutError: + return False + def domain_to_url(domain: str) -> str: """Convert a partial domain to valid URL. @@ -55,24 +78,4 @@ def login_using_cookie_file(driver: WebDriver, cookie_file: str) -> None: try: driver.add_cookie(cookie) except Exception: - print(f"Couldn't set cookie {cookie['name']} for {domain}") - - -def confirm_logged_in(driver: WebDriver) -> bool: - """Confirm that the user is logged in. - - The browser needs to be navigated to a YouTube page. - - Args: - driver: Selenium webdrive. - - Returns: - `True` if the user is logged in, otherwise `False`. - """ - try: - WebDriverWait(driver, 5).until( - ec.element_to_be_clickable((By.ID, "avatar-btn")) - ) - return True - except TimeoutError: - return False + logger.info(f"Couldn't set cookie {cookie['name']} for {domain}") diff --git a/publish/upload.py b/src/rybo/publish/upload.py similarity index 90% rename from publish/upload.py rename to src/rybo/publish/upload.py index 8aacb76..be3fd69 100644 --- a/publish/upload.py +++ b/src/rybo/publish/upload.py @@ -2,8 +2,11 @@ import logging import re from datetime import datetime, timedelta +from pathlib import Path from time import sleep +import colorama +from colorama import Fore from selenium.common.exceptions import ( ElementNotInteractableException, NoSuchElementException, @@ -15,107 +18,55 @@ from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait +logger = logging.getLogger(__name__) today = datetime.now() + timedelta(minutes=15) +colorama.init(autoreset=True) -def upload_file( - driver: WebDriver, - video_path: str, - title: str, - description: str, - game: str = False, - kids: bool = False, - upload_time: datetime = None, - thumbnail_path: str = None, + +def _set_advanced_settings( + driver: WebDriver, game_title: str, made_for_kids: bool ) -> None: - """Upload a single video to YouTube. + """Associate the video with a game, and/or flag it as suitable for kids. Args: driver: Selenium webdriver. - video_path: Path to the video to be uploaded. - title: Video title as it will appear on YouTube. - description: Video description as it will appear on YouTube. - game: Game that the video is associated with. - kids: `True` to indicate that the video is suitable for viewing by - kids, otherwise `false`. - upload_time: Date and time that the video was uploaded. + game_title: Name of the game that the video is relevant to. + made_for_kids: `True` to indicate the video is suitable for viewing by + kids, otherwise `False`. """ - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable((By.CSS_SELecTOR, "ytcp-button#create-icon")) - ).click() - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - (By.XPATH, '//tp-yt-paper-item[@test-id="upload-beta"]') + logger.info("Setting Advanced Settings") + # Open advanced options + driver.find_element(By.CSS_SELecTOR, "#toggle-button").click() + if game_title: + game_title_input: WebElement = driver.find_element( + By.CSS_SELecTOR, + ".ytcp-form-gaming > " + "ytcp-dropdown-trigger:nth-child(1) > " + ":nth-child(2) > div:nth-child(3) > input:nth-child(3)", ) - ).click() - video_input = driver.find_element(by=By.XPATH, value='//input[@type="file"]') - logging.info("Setting Video File Path") - video_input.send_keys(video_path) + game_title_input.send_keys(game_title) - _set_basic_settings(driver, title, description, thumbnail_path) - _set_advanced_settings(driver, game, kids) - # Go to visibility settings - for _i in range(3): + # Select first item in game drop down WebDriverWait(driver, 20).until( - ec.element_to_be_clickable((By.ID, "next-button")) + ec.element_to_be_clickable( + ( + By.CSS_SELecTOR, + "#text-item-2", # The first item is an empty item + ) + ) ).click() - # _set_time(driver, upload_time) - try: - _set_visibility(driver) - except Exception: - print("error uploading, continuing") - _wait_for_processing(driver) - # Go back to endcard settings - # find_element(By.CSS_SELecTOR,"#step-badge-1").click() - # _set_endcard(driver) - - # for _ in range(2): - # # Sometimes, the button is clickable but clicking it raises an - # # error, so we add a "safety-sleep" here - # sleep(5) - # WebDriverWait(driver, 20) - # .until(ec.element_to_be_clickable((By.ID, "next-button"))) - # .click() - - # sleep(5) - # WebDriverWait(driver, 20) - # .until(ec.element_to_be_clickable((By.ID, "done-button"))) - # .click() - - # # Wait for the dialog to disappear - # sleep(5) - driver.close() - logging.info("Upload is complete") - - -def _wait_for_processing(driver: WebDriver) -> None: - """Wait for YouTube to process the video. - - Calling this method will cause progress updates to be sent to stdout - every 5 seconds until the processing is complete. - - Args: - driver: Selenium webdriver. - """ - logging.info("Waiting for processing to complete") - # Wait for processing to complete - progress_label: WebElement = driver.find_element( - By.CSS_SELecTOR, "span.progress-label" - ) - pattern = re.compile(r"(finished processing)|(processing hd.*)|(check.*)") - current_progress = progress_label.get_attribute("textContent") - last_progress = None - while not pattern.match(current_progress.lower()): - if last_progress != current_progress: - logging.info(f"Current progress: {current_progress}") - last_progress = current_progress - sleep(5) - current_progress = progress_label.get_attribute("textContent") - if "Processing 99" in current_progress: - print("Finished Processing!") - sleep(10) - break + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable( + ( + By.NAME, + "VIDEO_MADE_FOR_KIDS_MFK" + if made_for_kids + else "VIDEO_MADE_FOR_KIDS_NOT_MFK", + ) + ) + ).click() def _set_basic_settings( @@ -133,7 +84,7 @@ def _set_basic_settings( thumbnail_path: Path to be used to set the videos thumbnail, as it appears in YouTube search results. """ - logging.info("Setting Basic Settings") + logger.info("Setting Basic Settings") title_input: WebElement = WebDriverWait(driver, 20).until( ec.element_to_be_clickable( ( @@ -157,67 +108,22 @@ def _set_basic_settings( ) title_input.clear() - logging.info("Setting Video Title") + logger.info("Setting Video Title") title_input.send_keys(title) - logging.info("Setting Video Description") + logger.info("Setting Video Description") description_input.send_keys(description) if thumbnail_path: - logging.info("Setting Video Thumbnail") + logger.info("Setting Video Thumbnail") thumbnail_input.send_keys(thumbnail_path) -def _set_advanced_settings( - driver: WebDriver, game_title: str, made_for_kids: bool -) -> None: - """Associate the video with a game, and/or flag it as suitable for kids. - - Args: - driver: Selenium webdriver. - game_title: Name of the game that the video is relevant to. - made_for_kids: `True` to indicate the video is suitable for viewing by - kids, otherwise `False`. - """ - logging.info("Setting Advanced Settings") - # Open advanced options - driver.find_element(By.CSS_SELecTOR, "#toggle-button").click() - if game_title: - game_title_input: WebElement = driver.find_element( - By.CSS_SELecTOR, - ".ytcp-form-gaming > " - "ytcp-dropdown-trigger:nth-child(1) > " - ":nth-child(2) > div:nth-child(3) > input:nth-child(3)", - ) - game_title_input.send_keys(game_title) - - # Select first item in game drop down - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - ( - By.CSS_SELecTOR, - "#text-item-2", # The first item is an empty item - ) - ) - ).click() - - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - ( - By.NAME, - "VIDEO_MADE_FOR_KIDS_MFK" - if made_for_kids - else "VIDEO_MADE_FOR_KIDS_NOT_MFK", - ) - ) - ).click() - - def _set_endcard(driver: WebDriver) -> None: """Set the end card. Args: driver: Selenium webdriver. """ - logging.info("Endscreen") + logger.info("Endscreen") # Add endscreen driver.find_element(By.CSS_SELecTOR, "#endscreens-button").click() @@ -229,7 +135,9 @@ def _set_endcard(driver: WebDriver) -> None: driver.find_element(By.CSS_SELecTOR, "div.card:nth-child(1)").click() break except (NoSuchElementException, ElementNotInteractableException): - logging.warning(f"Couldn't find endcard button. Retry in 5s! ({i}/10)") + logger.warning( + f"{Fore.YELLOW}Couldn't find endcard button. Retry in 5s! ({i}/10)" + ) sleep(5) WebDriverWait(driver, 20).until( @@ -284,7 +192,7 @@ def _set_visibility(driver: WebDriver) -> None: driver: Selenium webdriver. """ # Start time scheduling - logging.info("Setting Visibility to public") + logger.info("Setting Visibility to public") WebDriverWait(driver, 30).until( ec.element_to_be_clickable((By.NAME, "FIRST_CONTAINER")) ).click() @@ -298,3 +206,104 @@ def _set_visibility(driver: WebDriver) -> None: # sleep(10) # WebDriverWait(driver, 30) # .until(ec.element_to_be_clickable((By.ID, "close-button"))).click() + + +def upload_file( + driver: WebDriver, + video_path: Path, + title: str, + description: str, + game: str = False, + kids: bool = False, + upload_time: datetime = None, + thumbnail_path: str = None, +) -> None: + """Upload a single video to YouTube. + + Args: + driver: Selenium webdriver. + video_path: Path to the video to be uploaded. + title: Video title as it will appear on YouTube. + description: Video description as it will appear on YouTube. + game: Game that the video is associated with. + kids: `True` to indicate that the video is suitable for viewing by + kids, otherwise `false`. + upload_time: Date and time that the video was uploaded. + thumbnail_path: Path to the thumbnail to be used. + """ + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable((By.CSS_SELecTOR, "ytcp-button#create-icon")) + ).click() + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable( + (By.XPATH, '//tp-yt-paper-item[@test-id="upload-beta"]') + ) + ).click() + video_input = driver.find_element(by=By.XPATH, value='//input[@type="file"]') + logger.info("Setting Video File Path") + video_input.send_keys(video_path) + + _set_basic_settings(driver, title, description, thumbnail_path) + _set_advanced_settings(driver, game, kids) + # Go to visibility settings + for _i in range(3): + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable((By.ID, "next-button")) + ).click() + + # _set_time(driver, upload_time) + try: + _set_visibility(driver) + except Exception: + logger.info("error uploading, continuing") + _wait_for_processing(driver) + # Go back to endcard settings + # find_element(By.CSS_SELecTOR,"#step-badge-1").click() + # _set_endcard(driver) + + # for _ in range(2): + # # Sometimes, the button is clickable but clicking it raises an + # # error, so we add a "safety-sleep" here + # sleep(5) + # WebDriverWait(driver, 20) + # .until(ec.element_to_be_clickable((By.ID, "next-button"))) + # .click() + + # sleep(5) + # WebDriverWait(driver, 20) + # .until(ec.element_to_be_clickable((By.ID, "done-button"))) + # .click() + + # # Wait for the dialog to disappear + # sleep(5) + driver.close() + logger.info("Upload is complete") + + +def _wait_for_processing(driver: WebDriver) -> None: + """Wait for YouTube to process the video. + + Calling this method will cause progress updates to be sent to stdout + every 5 seconds until the processing is complete. + + Args: + driver: Selenium webdriver. + """ + logger.info("Waiting for processing to complete") + # Wait for processing to complete + progress_label: WebElement = driver.find_element( + By.CSS_SELecTOR, "span.progress-label" + ) + pattern = re.compile(r"(finished processing)|(processing hd.*)|(check.*)") + current_progress = progress_label.get_attribute("textContent") + last_progress = None + while not pattern.match(current_progress.lower()): + if last_progress != current_progress: + logger.info(f"Current progress: {current_progress}") + last_progress = current_progress + sleep(5) + current_progress = progress_label.get_attribute("textContent") + if "Processing 99" in current_progress: + logger.info("Finished Processing!") + sleep(10) + break diff --git a/publish/youtube.py b/src/rybo/publish/youtube.py similarity index 78% rename from publish/youtube.py rename to src/rybo/publish/youtube.py index 7ef2e99..40b6d39 100644 --- a/publish/youtube.py +++ b/src/rybo/publish/youtube.py @@ -6,16 +6,13 @@ from simple_youtube_api.Channel import Channel from simple_youtube_api.LocalVideo import LocalVideo -#from simple_youtube_api.youtube_video import YouTubeVideo -import config.settings as settings +from rybo.config import settings -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log"), logging.StreamHandler()], -) +# from simple_youtube_api.youtube_video import YouTubeVideo + + +logger = logging.getLogger(__name__) @dataclass @@ -34,10 +31,10 @@ def publish(video: Video) -> None: Args: video: Metadata describing the video to be uploaded. """ - logging.info("========== Uploading Video To Youtube ==========") - logging.info("video.filepath : %s", video.filepath) - logging.info("video.title : %s", video.title) - logging.info("video.thumbnail : %s", video.thumbnail) + logger.info("========== Uploading Video To Youtube ==========") + logger.info("video.filepath : %s", video.filepath) + logger.info("video.title : %s", video.title) + logger.info("video.thumbnail : %s", video.thumbnail) # loggin into the channel channel: Channel = Channel() @@ -65,11 +62,11 @@ def publish(video: Video) -> None: try: # uploading video and printing the results uploaded_video = channel.upload_video(youtube_upload) - print(uploaded_video.id) - print(uploaded_video) + logger.info(uploaded_video.id) + logger.info(uploaded_video) except Exception as e: - logging.info("Error uploading video : %s", video.title) - print(e) + logger.info("Error uploading video : %s", video.title) + logger.info(e) if __name__ == "__main__": diff --git a/src/rybo/reddit/__init__.py b/src/rybo/reddit/__init__.py new file mode 100644 index 0000000..972def6 --- /dev/null +++ b/src/rybo/reddit/__init__.py @@ -0,0 +1 @@ +"""Reddit post processors.""" diff --git a/reddit/reddit.py b/src/rybo/reddit/reddit.py similarity index 65% rename from reddit/reddit.py rename to src/rybo/reddit/reddit.py index 2de57a3..3b03eda 100644 --- a/reddit/reddit.py +++ b/src/rybo/reddit/reddit.py @@ -1,157 +1,90 @@ """Helpers used to retreive, filter and process reddit posts.""" -import praw -import config.settings as settings -import config.auth as auth import base64 +import logging import re -from praw.models import Submission from typing import List +import praw +from praw.models import Submission -def is_valid_submission(submission) -> bool: - """Determine whether a Reddit post is worth turning in to a video. +from rybo.config import settings - A post is deemed "worthy", if: - - It isn't stickied. - - It was submitted by ttvibe. - - The posts title is within the min/max ranges defined in settings. - - The post hasn't been flagged as NSFW. - - The post doesn't contain banned keywords. - - The post wasn't made in a subreddit that has been added to the - ignore list. - - The submission score is within range. - - The length of the post content is less than the configured maximum - length. - - The post has more than a minimum number of comments. - - The post is not an update on a previous post. - - The post doesn't contain 'covid' or 'vaccine' in the title, as these - tend to trigger Youtube strikes. Censorship is double-plus good... +logger = logging.getLogger(__name__) + + +def get_reddit_mentions(client_id: str, client_secret: str) -> List[str]: + """Get a list of comments where ttvibe has been mentioned. Args: - submission: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. Returns: - `True` if the Reddit post is deemed worthy of turning in to a video, - otherwise returns `False`. + A list containing zero or more URLs where ttvibe has been mentioned. """ - if submission.stickied: - return False - - if not settings.enable_screenshot_title_image and not submission.is_self: - return False - - if ( - len(submission.title) < settings.title_length_minimum - or len(submission.title) > settings.title_length_maximum - ): - return False - - if not settings.enable_nsfw_content: - if submission.over_18: - print("Skipping NSFW...") - return False - for banned_keyword in ( - base64.b64decode(settings.banned_keywords_base64.encode("ascii")) - .decode("ascii") - .split(",") - ): - if banned_keyword in submission.title.lower(): - print( - f"{submission.title} \ - <-- Skipping post, title contains banned keyword!" - ) - return False - - if submission.subreddit_name_prefixed in settings.subreddits_excluded: - return False - - if submission.score < settings.minimum_submission_score: - print(f"{submission.title} <-- Submission score too low!") - return False - - if len(submission.selftext) > settings.maximum_length_self_text: - return False - - if ( - settings.enable_comments - and submission.num_comments < settings.minimum_num_comments - ): - print(f"{submission.title} <-- Number of comments too low!") - return False - - if "update" in submission.title.lower(): - return False + r: praw.Reddit = praw.Reddit( + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 \ + Firefox/112.0", + ) - if "covid" in submission.title.lower() or "vaccine" in submission.title.lower(): - print(f"{submission.title} <-- Youtube Channel Strikes if Covid content...!") - return False + mention_urls: List[str] = [] + for mention in r.inbox.mentions(limit=None): + post_url: str = re.sub( + rf"/{mention.id}/\?context=\d", "", mention.context, flags=re.IGNORECASE + ) + mention_urls.append(f"https://www.reddit.com{post_url}") - return True + return mention_urls -def get_reddit_submission(url: str) -> Submission: +def get_reddit_submission(url: str, client_id: str, client_secret: str) -> Submission: """Get a single Reddit post. Args: url: URL to the post to be retrieved. + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. Returns: The post contents. """ r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 \ + Firefox/112.0", ) submission: Submission = r.submission(url=url) return submission -def get_reddit_mentions() -> List[str]: - """Get a list of comments where ttvibe has been mentioned. - - Returns: - A list containing zero or more URLs where ttvibe has been mentioned. - """ - r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, - username=auth.praw_username, - password=auth.praw_password, - ) - - mention_urls: List[str] = [] - for mention in r.inbox.mentions(limit=None): - post_url: str = re.sub( - rf"/{mention.id}/\?context=\d", "", mention.context, flags=re.IGNORECASE - ) - mention_urls.append(f"https://www.reddit.com{post_url}") - - return mention_urls - - -def get_reddit_submissions() -> List[Submission]: +def get_reddit_submissions(client_id: str, client_secret: str) -> List[Submission]: """Get the latest Reddit posts. Posts will be retrieved according to the sort order defined in settings. + Args: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. + Returns: A list containing zero or more Reddit posts. """ r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101\ + Firefox/112.0", ) if settings.subreddits: subreddits: str = "+".join(settings.subreddits) else: subreddits: str = "all" - print("Retrieving posts from subreddit :") - print(subreddits) + logger.info("Retrieving posts from subreddit %s", subreddits) # Get Reddit Hot Posts if settings.reddit_post_sort == "hot": @@ -182,29 +115,120 @@ def get_valid_submissions(submissions: List[Submission]) -> List[Submission]: material, to generate a video. """ valid_submissions: List[Submission] = [] - print("===== Retrieving valid Reddit submissions =====") - print("ID, SCORE, NUM_COMMENTS, LEN_SELFTEXT, SUBREDDIT, TITLE") + logger.info("===== Retrieving valid Reddit submissions =====") + logger.info("ID, SCORE, NUM_COMMENTS, LEN_SELFTEXT, SUBREDDIT, TITLE") for submission in submissions: if is_valid_submission(submission): msg: str = ", ".join( - [str(submission.id), - str(submission.score), - str(submission.num_comments), - str(len(submission.selftext)), - submission.subreddit_name_prefixed, - submission.title] + [ + str(submission.id), + str(submission.score), + str(submission.num_comments), + str(len(submission.selftext)), + submission.subreddit_name_prefixed, + submission.title, + ] ) - print(msg) + logger.info(msg) valid_submissions.append(submission) return valid_submissions -def posts() -> List[Submission]: +def is_valid_submission(submission: Submission) -> bool: + """Determine whether a Reddit post is worth turning in to a video. + + A post is deemed "worthy", if: + - It isn't stickied. + - It was submitted by ttvibe. + - The posts title is within the min/max ranges defined in settings. + - The post hasn't been flagged as NSFW. + - The post doesn't contain banned keywords. + - The post wasn't made in a subreddit that has been added to the + ignore list. + - The submission score is within range. + - The length of the post content is less than the configured maximum + length. + - The post has more than a minimum number of comments. + - The post is not an update on a previous post. + - The post doesn't contain 'covid' or 'vaccine' in the title, as these + tend to trigger Youtube strikes. Censorship is double-plus good... + + Args: + submission: A single Reddit post. + + Returns: + `True` if the Reddit post is deemed worthy of turning in to a video, + otherwise returns `False`. + """ + if submission.stickied: + return False + + if not settings.enable_screenshot_title_image and not submission.is_self: + return False + + if ( + len(submission.title) < settings.title_length_minimum + or len(submission.title) > settings.title_length_maximum + ): + return False + + if not settings.enable_nsfw_content: + if submission.over_18: + logger.info("Skipping NSFW...") + return False + for banned_keyword in ( + base64.b64decode(settings.banned_keywords_base64.encode("ascii")) + .decode("ascii") + .split(",") + ): + if banned_keyword in submission.title.lower(): + logger.info( + f"{submission.title} \ + <-- Skipping post, title contains banned keyword!" + ) + return False + + if submission.subreddit_name_prefixed in settings.subreddits_excluded: + return False + + if submission.score < settings.minimum_submission_score: + logger.info(f"{submission.title} <-- Submission score too low!") + return False + + if len(submission.selftext) > settings.maximum_length_self_text: + return False + + if ( + settings.enable_comments + and submission.num_comments < settings.minimum_num_comments + ): + logger.info(f"{submission.title} <-- Number of comments too low!") + return False + + if "update" in submission.title.lower(): + return False + + if "covid" in submission.title.lower() or "vaccine" in submission.title.lower(): + logger.info( + f"{submission.title} <-- Youtube Channel Strikes if Covid content...!" + ) + return False + + return True + + +def posts(client_id: str, client_secret: str) -> List[Submission]: """Get a list of available Reddit posts. + Args: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. + Returns: A list of zero or more Reddit submissions. """ - submissions: List[Submission] = get_reddit_submissions() + submissions: List[Submission] = get_reddit_submissions( + client_id=client_id, client_secret=client_secret + ) return submissions diff --git a/src/rybo/speech/__init__.py b/src/rybo/speech/__init__.py new file mode 100644 index 0000000..5b54a62 --- /dev/null +++ b/src/rybo/speech/__init__.py @@ -0,0 +1 @@ +"""Text to Speech processors.""" diff --git a/speech/speech.py b/src/rybo/speech/speech.py similarity index 86% rename from speech/speech.py rename to src/rybo/speech/speech.py index acebf6f..3e5cafe 100644 --- a/speech/speech.py +++ b/src/rybo/speech/speech.py @@ -12,69 +12,13 @@ from gtts import gTTS from moviepy.editor import AudioFileClip, concatenate_audioclips -import config -import config.settings as settings -from speech.streamlabs_polly import StreamlabsPolly -from speech.tiktok import TikTok -from utils.common import sanitize_text +from rybo import config +from rybo.config import settings +from rybo.speech.streamlabs_polly import StreamlabsPolly +from rybo.speech.tiktok import TikTok +from rybo.utils.common import sanitize_text -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) - - -def process_speech_text(text: str) -> str: - """Sanitize raw text. - - This will process raw text prior to converting it to speech, replacing - common abbreviations with their full english version to ensure that the - generated speech is intelligable. - - Args: - text: Text to be sanitized. - - Returns: - Updated text that has had common abbreviations converted to their - full english equivalent. - """ - text = text.replace(" AFAIK ", " as far as I know ") - text = text.replace("AITA", " Am I The Asshole? ") - text = text.replace(" AMA ", " Ask me anything ") - text = text.replace(" ELI5 ", " Explain Like I'm Five ") - text = text.replace(" IAMA ", " I am a ") - text = text.replace("IANAD", " i am not a doctor ") - text = text.replace("IANAL", " i am not a lawyer ") - text = text.replace(" IMO ", " in my opinion ") - text = text.replace(" NSFL ", " Not safe for life ") - text = text.replace(" NSFW ", " Not safe for Work ") - text = text.replace("NTA", " Not The Asshole ") - text = text.replace(" SMH ", " Shaking my head ") - text = text.replace("TD;LR", " too long didn't read ") - text = text.replace("TDLR", " too long didn't read ") - text = text.replace(" TIL ", " Today I Learned ") - text = text.replace("YTA", " You're the asshole ") - text = text.replace("SAHM", " stay at home mother ") - text = text.replace("WIBTA", " would I be the asshole ") - text = text.replace(" stfu ", " shut the fuck up ") - text = text.replace(" OP ", " o p ") - text = text.replace(" CB ", " choosing beggar ") - text = text.replace("pettyrevenge", "petty revenge") - text = text.replace("askreddit", "ask reddit") - text = text.replace("twoxchromosomes", "two x chromosomes") - text = text.replace("showerthoughts", "shower thoughts") - text = text.replace("amitheasshole", "am i the asshole") - text = text.replace("“", '"') - text = text.replace("“", '"') - text = text.replace("’", "'") - text = text.replace("...", ".") - text = text.replace("*", "") - text = re.sub(r"(\[|\()[0-9]{1,2}\s*(m|f)?(\)|\])", "", text, flags=re.IGNORECASE) - - text = sanitize_text(text) - return text +logger = logging.getLogger(__name__) def create_audio(path: Path, text: str) -> Path: @@ -87,7 +31,7 @@ def create_audio(path: Path, text: str) -> Path: Returns: Path to the generated audio file. """ - # logging.info(f"Generating Audio File : {text}") + # logger.info(f"Generating Audio File : {text}") output_path = os.path.normpath(path) text: str = process_speech_text(text) if not os.path.exists(output_path) or not os.path.getsize(output_path) > 0: @@ -114,7 +58,7 @@ def create_audio(path: Path, text: str) -> Path: speech_text_character_limit: int = 550 if len(text) > speech_text_character_limit: - logging.info( + logger.info( "Text exceeds StreamlabsPolly limit, breaking up into chunks" ) speech_chunks: List[Path] = [] @@ -124,16 +68,16 @@ def create_audio(path: Path, text: str) -> Path: break_long_words=True, break_on_hyphens=False, ) - print(chunk_list) + logger.info(chunk_list) for count, chunk in enumerate(chunk_list): - print(count) + logger.info(count) if chunk == "​": - logging.info("Skip zero space character comment : %s", chunk) + logger.info("Skip zero space character comment : %s", chunk) continue if chunk == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue tmp_path: Path = f"{output_path}{count}" @@ -144,7 +88,7 @@ def create_audio(path: Path, text: str) -> Path: final_clip: AudioFileClip = concatenate_audioclips(clips) final_clip.write_audiofile(output_path) else: - print(text) + logger.info(text) slp.run(text, output_path) if settings.voice_engine == "edge-tts": @@ -169,7 +113,7 @@ def create_audio(path: Path, text: str) -> Path: tt: TikTok = TikTok() if len(text) > speech_text_character_limit: - logging.info( + logger.info( "Text exceeds tiktok limit, \ breaking up into chunks" ) @@ -180,16 +124,16 @@ def create_audio(path: Path, text: str) -> Path: break_long_words=True, break_on_hyphens=False, ) - print(chunk_list) + logger.info(chunk_list) for count, chunk in enumerate(chunk_list): - print(count) + logger.info(count) if chunk == "​": - logging.info("Skip zero space character comment : %s", chunk) + logger.info("Skip zero space character comment : %s", chunk) continue if chunk == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue tmp_path: Path = f"{path}{count}" @@ -200,16 +144,67 @@ def create_audio(path: Path, text: str) -> Path: final_clip: AudioFileClip = concatenate_audioclips(clips) final_clip.write_audiofile(output_path) else: - print(text) + logger.info(text) tt.run(text, output_path) else: - logging.info("Audio file already exists : %s", output_path) + logger.info("Audio file already exists : %s", output_path) - logging.info("Created Audio File : %s", output_path) + logger.info("Created Audio File : %s", output_path) return output_path +def process_speech_text(text: str) -> str: + """Sanitize raw text. + + This will process raw text prior to converting it to speech, replacing + common abbreviations with their full english version to ensure that the + generated speech is intelligable. + + Args: + text: Text to be sanitized. + + Returns: + Updated text that has had common abbreviations converted to their + full english equivalent. + """ + text = text.replace(" AFAIK ", " as far as I know ") + text = text.replace("AITA", " Am I The Asshole? ") + text = text.replace(" AMA ", " Ask me anything ") + text = text.replace(" ELI5 ", " Explain Like I'm Five ") + text = text.replace(" IAMA ", " I am a ") + text = text.replace("IANAD", " i am not a doctor ") + text = text.replace("IANAL", " i am not a lawyer ") + text = text.replace(" IMO ", " in my opinion ") + text = text.replace(" NSFL ", " Not safe for life ") + text = text.replace(" NSFW ", " Not safe for Work ") + text = text.replace("NTA", " Not The Asshole ") + text = text.replace(" SMH ", " Shaking my head ") + text = text.replace("TD;LR", " too long didn't read ") + text = text.replace("TDLR", " too long didn't read ") + text = text.replace(" TIL ", " Today I Learned ") + text = text.replace("YTA", " You're the asshole ") + text = text.replace("SAHM", " stay at home mother ") + text = text.replace("WIBTA", " would I be the asshole ") + text = text.replace(" stfu ", " shut the fuck up ") + text = text.replace(" OP ", " o p ") + text = text.replace(" CB ", " choosing beggar ") + text = text.replace("pettyrevenge", "petty revenge") + text = text.replace("askreddit", "ask reddit") + text = text.replace("twoxchromosomes", "two x chromosomes") + text = text.replace("showerthoughts", "shower thoughts") + text = text.replace("amitheasshole", "am i the asshole") + text = text.replace("“", '"') + text = text.replace("“", '"') + text = text.replace("’", "'") + text = text.replace("...", ".") + text = text.replace("*", "") + text = re.sub(r"(\[|\()[0-9]{1,2}\s*(m|f)?(\)|\])", "", text, flags=re.IGNORECASE) + + text = sanitize_text(text) + return text + + if __name__ == "__main__": parser: ArgumentParser = ArgumentParser() parser.add_argument( @@ -220,6 +215,6 @@ def create_audio(path: Path, text: str) -> Path: ) args: Namespace = parser.parse_args() - print(args.path) - print(args.speech) + logger.info(args.path) + logger.info(args.speech) create_audio(args.path, args.speech) diff --git a/speech/streamlabs_polly.py b/src/rybo/speech/streamlabs_polly.py similarity index 94% rename from speech/streamlabs_polly.py rename to src/rybo/speech/streamlabs_polly.py index ae7a261..7fe71da 100644 --- a/speech/streamlabs_polly.py +++ b/src/rybo/speech/streamlabs_polly.py @@ -1,4 +1,5 @@ """Polly text to speech convertor.""" +import logging import sys import time as pytime from datetime import datetime @@ -8,14 +9,16 @@ from typing import Dict, List, Union import requests +from requests import Response +from requests.exceptions import JSONDecodeError + +from rybo.config import settings +from rybo.utils.common import sanitize_text # from utils import settings # from utils.voice import check_ratelimit -from requests import Response -from requests.exceptions import JSONDecodeError -import config.settings as settings -from utils.common import sanitize_text +logger = logging.getLogger(__name__) if sys.version_info[0] >= 3: from datetime import timezone @@ -40,67 +43,6 @@ # valid voices https://lazypy.ro/tts/ -def sleep_until(time: Union[int, datetime]) -> None: - """Pause until a specific end time. - - Args: - time: Either a valid datetime object or unix timestamp in seconds - (i.e. seconds since Unix epoch). - """ - end: int = time - - # Convert datetime to unix timestamp and adjust for locality - if isinstance(time, datetime): - # If we're on Python 3 and the user specified a timezone, convert to - # UTC and get the timestamp. - if sys.version_info[0] >= 3 and time.tzinfo: - end: datetime = time.astimezone(timezone.utc).timestamp() - else: - zone_diff: float = ( - pytime.time() - (datetime.now() - datetime(1970, 1, 1)).total_seconds() - ) - end: float = (time - datetime(1970, 1, 1)).total_seconds() + zone_diff - - # Type check - if not isinstance(end, (int, float)): - raise Exception("The time parameter is not a number or datetime object") - - # Now we wait - while True: - now: float = pytime.time() - diff: float = end - now - - # Time is up! - if diff <= 0: - break - else: - # 'logarithmic' sleeping to minimize loop iterations - sleep(diff / 2) - - -def check_ratelimit(response: Response) -> bool: - """Check if the rate limit has been hit. - - If it has, sleep for the time specified in the response. - - Args: - response: The HTTP response to be examined. - - Returns: - `True` if the rate limit has been reached, otherwise `False`. - """ - if response.status_code == 429: - try: - time: int = int(response.headers["X-RateLimit-Reset"]) - print("Ratelimit hit, sleeping...") - sleep_until(time) - return False - except KeyError: # if the header is not present, we don't know how long to wait - return False - - return True - - class StreamlabsPolly: """Polly text to speech convertor.""" @@ -110,6 +52,15 @@ def __init__(self): self.max_chars: int = 550 self.voices: List[str] = voices + def randomvoice(self) -> str: + """Select a random Polly voice. + + Returns: + The name of a randomly selected Polly voice. + """ + rnd: SystemRandom() = SystemRandom() + return rnd.choice(self.voices) + def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: """Convert text to an audio clip using a random Polly voice. @@ -145,13 +96,65 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: if response.json()["error"] == "No text specified!": raise ValueError("Please specify a text to convert to speech.") except (KeyError, JSONDecodeError): - print("Error occurred calling Streamlabs Polly") + logger.info("Error occurred calling Streamlabs Polly") - def randomvoice(self) -> str: - """Select a random Polly voice. - Returns: - The name of a randomly selected Polly voice. - """ - rnd: SystemRandom() = SystemRandom() - return rnd.choice(self.voices) +def check_ratelimit(response: Response) -> bool: + """Check if the rate limit has been hit. + + If it has, sleep for the time specified in the response. + + Args: + response: The HTTP response to be examined. + + Returns: + `True` if the rate limit has been reached, otherwise `False`. + """ + if response.status_code == 429: + try: + time: int = int(response.headers["X-RateLimit-Reset"]) + logger.info("Ratelimit hit, sleeping...") + sleep_until(time) + return False + except KeyError: # if the header is not present, we don't know how long to wait + return False + + return True + + +def sleep_until(time: Union[int, datetime]) -> None: + """Pause until a specific end time. + + Args: + time: Either a valid datetime object or unix timestamp in seconds + (i.e. seconds since Unix epoch). + """ + end: int = time + + # Convert datetime to unix timestamp and adjust for locality + if isinstance(time, datetime): + # If we're on Python 3 and the user specified a timezone, convert to + # UTC and get the timestamp. + if sys.version_info[0] >= 3 and time.tzinfo: + end: datetime = time.astimezone(timezone.utc).timestamp() + else: + zone_diff: float = ( + pytime.time() - (datetime.now() - datetime(1970, 1, 1)).total_seconds() + ) + end: float = (time - datetime(1970, 1, 1)).total_seconds() + zone_diff + + # Type check + if not isinstance(end, (int, float)): + raise Exception("The time parameter is not a number or datetime object") + + # Now we wait + while True: + now: float = pytime.time() + diff: float = end - now + + # Time is up! + if diff <= 0: + break + else: + # 'logarithmic' sleeping to minimize loop iterations + sleep(diff / 2) diff --git a/speech/tiktok.py b/src/rybo/speech/tiktok.py similarity index 95% rename from speech/tiktok.py rename to src/rybo/speech/tiktok.py index 162091c..aff2a1a 100644 --- a/speech/tiktok.py +++ b/src/rybo/speech/tiktok.py @@ -1,8 +1,9 @@ """TikTok text to speech convertor.""" import base64 -from random import SystemRandom +import logging import urllib.parse from pathlib import Path +from random import SystemRandom from typing import Dict, List import requests @@ -10,7 +11,9 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import SSLError -import config.settings as settings +from rybo.config import settings + +logger = logging.getLogger(__name__) # from profanity_filter import ProfanityFilter # pf = ProfanityFilter() @@ -87,6 +90,15 @@ def __init__(self): "noneng": noneng, } + def randomvoice(self) -> str: + """Select a random voice. + + Returns: + A randomly chosen human voice. + """ + rnd: SystemRandom = SystemRandom() + return rnd.choice(self.voices["human"]) + def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: """Convert text to an audio clip using a random tik-tok styled voice. @@ -107,9 +119,9 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: ) try: text: str = urllib.parse.quote(text) - print(len(text)) + logger.info(len(text)) tt_uri: str = f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" - print(tt_uri) + logger.info(tt_uri) r: Response = requests.post(tt_uri, timeout=120) except SSLError: # https://stackoverflow.com/a/47475019/18516611 @@ -121,25 +133,16 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: r: Response = session.post( f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" ) - # print(r.text) + # logger.info(r.text) vstr: str = [r.json()["data"]["v_str"]][0] b64d: bytes = base64.b64decode(vstr) with open(filepath, "wb") as out: # noqa: SCS109 out.write(b64d) - def randomvoice(self) -> str: - """Select a random voice. - - Returns: - A randomly chosen human voice. - """ - rnd: SystemRandom = SystemRandom() - return rnd.choice(self.voices["human"]) - if __name__ == "__main__": tt: TikTok = TikTok() text_to_say: str = "Hello world this is some spoken text" - print(str(len(text_to_say))) + logger.info(str(len(text_to_say))) tt.run(text_to_say, "tiktok_test.mp3") diff --git a/src/rybo/thumbnail/__init__.py b/src/rybo/thumbnail/__init__.py new file mode 100644 index 0000000..b18e605 --- /dev/null +++ b/src/rybo/thumbnail/__init__.py @@ -0,0 +1 @@ +"""Thumbnail generators.""" diff --git a/thumbnail/keywords.py b/src/rybo/thumbnail/keywords.py similarity index 79% rename from thumbnail/keywords.py rename to src/rybo/thumbnail/keywords.py index a53906d..c94605b 100644 --- a/thumbnail/keywords.py +++ b/src/rybo/thumbnail/keywords.py @@ -1,8 +1,13 @@ """Functions to extract key phrases out of text.""" +import logging +from typing import List + from multi_rake import Rake +logger = logging.getLogger(__name__) + -def get_keywords(text): +def get_keywords(text) -> List[str]: """Extract key phrases out of a block of text. Args: @@ -11,8 +16,8 @@ def get_keywords(text): Returns: A selection of key phrases. """ - rake = Rake() - keywords = rake.apply(text) + rake: Rake = Rake() + keywords: List[str] = rake.apply(text) return keywords @@ -29,5 +34,5 @@ def get_keywords(text): "systems and systems of mixed types." ) - keywords = get_keywords(text) - print(keywords[:10]) + keywords: List[str] = get_keywords(text) + logger.info(keywords[:10]) diff --git a/thumbnail/lexica.py b/src/rybo/thumbnail/lexica.py similarity index 85% rename from thumbnail/lexica.py rename to src/rybo/thumbnail/lexica.py index 30eeebd..5357e74 100644 --- a/thumbnail/lexica.py +++ b/src/rybo/thumbnail/lexica.py @@ -7,10 +7,13 @@ from pathlib import Path from typing import List from urllib.request import Request, urlopen -from requests import Response + import requests +from requests import Response + +from rybo.config import settings -import config.settings as settings +logger = logging.getLogger(__name__) def download_image(url: str, file_path: Path) -> None: @@ -51,12 +54,12 @@ def get_images( if number_of_images > 0: safe_query: str = urllib.parse.quote(sentence.strip()) lexica_url: str = f"https://lexica.art/api/v1/search?q={safe_query}" - logging.info("Downloading Image From Lexica : %s", sentence) + logger.info("Downloading Image From Lexica : %s", sentence) try: r: Response = requests.get(lexica_url, timeout=120) j: object = json.loads(r.text) except Exception: - print("Error Retrieving Lexica Images") + logger.info("Error Retrieving Lexica Images") pass return @@ -66,9 +69,9 @@ def get_images( image_url: str = j["images"][num]["src"] download_image(image_url, image_path) else: - logging.info("Image already exists : %s - %s", id, sentence) + logger.info("Image already exists : %s - %s", id, sentence) images.append(image_path) else: - logging.info("Downloading Images Set to False.......") + logger.info("Downloading Images Set to False.......") return images diff --git a/thumbnail/thumbnail.py b/src/rybo/thumbnail/thumbnail.py similarity index 92% rename from thumbnail/thumbnail.py rename to src/rybo/thumbnail/thumbnail.py index eec871c..922602a 100644 --- a/thumbnail/thumbnail.py +++ b/src/rybo/thumbnail/thumbnail.py @@ -7,24 +7,20 @@ import os import sys from pathlib import Path - -# from nltk.corpus import stopwords from random import SystemRandom from typing import Any, List from moviepy.editor import CompositeVideoClip, ImageClip, TextClip from PIL import Image -import config.settings as settings -import thumbnail.lexica as lexica -from utils.common import random_rgb_colour, sanitize_text +from rybo.config import settings +from rybo.thumbnail import lexica +from rybo.utils.common import random_rgb_colour, sanitize_text + +# from nltk.corpus import stopwords + -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) +logger = logging.getLogger(__name__) def apply_black_gradient( @@ -64,7 +60,7 @@ def apply_black_gradient( alpha_gradient.putpixel((x, 0), a) else: alpha_gradient.putpixel((x, 0), 0) - # print '{}, {:.2f}, {}'.format(x, float(x) / width, a) + # logger.info '{}, {:.2f}, {}'.format(x, float(x) / width, a) alpha: Any = alpha_gradient.resize(input_im.size) # create black image, apply gradient @@ -87,8 +83,8 @@ def get_font_size(length: int) -> int: Returns: A suggested font size. """ - fontsize = 50 - lineheight = 60 + fontsize: int = 50 + lineheight: int = 60 if length < 10: fontsize = 190 @@ -120,9 +116,9 @@ def get_font_size(length: int) -> int: if length >= 90 and length < 100: fontsize = 80 - logging.debug("Title Length : %s", length) - logging.debug("Setting Fontsize : %s", fontsize) - logging.debug("Setting Lineheight : %s", lineheight) + logger.debug("Title Length : %s", length) + logger.debug("Setting Fontsize : %s", fontsize) + logger.debug("Setting Lineheight : %s", lineheight) return fontsize, lineheight @@ -148,7 +144,7 @@ def generate( Returns: A list of paths where the downloaded images can be found. """ - logging.info("========== Generating Thumbnail ==========") + logger.info("========== Generating Thumbnail ==========") # image_path = str(Path(video_directory, "lexica.png").absolute()) @@ -158,7 +154,7 @@ def generate( title = title.replace(" ", " ") title = title.replace("’", "") - logging.info(title) + logger.info(title) if settings.thumbnail_image_source == "random": rnd: SystemRandom = SystemRandom() @@ -173,7 +169,7 @@ def generate( video_directory, title, number_of_images=number_of_thumbnails ) - thumbnails = [] + thumbnails: List[Path] = [] if images: for index, image in enumerate(images): @@ -206,7 +202,7 @@ def create_thumbnail( video_directory, f"thumbnail_{str(index)}.png" ).absolute() if thumbnail_path.exists(): - logging.info("Thumbnail already exists : %s", thumbnail_path) + logger.info("Thumbnail already exists : %s", thumbnail_path) return thumbnail_path border_width: int = 20 @@ -271,8 +267,8 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: # fontsize = 40 title = title.replace("’", "") fontsize, lineheight = get_font_size(len(title)) - logging.info("Title Length : %s", len(title)) - logging.info("Optimising Font Size : ") + logger.info("Title Length : %s", len(title)) + logger.info("Optimising Font Size : ") sys.stdout.write(str(fontsize)) while True: @@ -282,7 +278,7 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: sys.stdout.write(".") if txt_clip.h > height: optimal_font_size: int = previous_fontsize - print(optimal_font_size) + logger.info(optimal_font_size) break word_height: TextClip = get_text_clip( @@ -377,7 +373,7 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: final_video: CompositeVideoClip = CompositeVideoClip(clips) final_video = final_video.margin(border_width, color=random_rgb_colour()) - logging.info("Saving Thumbnail to : %s", thumbnail_path) + logger.info("Saving Thumbnail to : %s", thumbnail_path) final_video.save_frame(thumbnail_path, 1) return thumbnail_path diff --git a/src/rybo/utils/__init__.py b/src/rybo/utils/__init__.py new file mode 100644 index 0000000..5c7ffd7 --- /dev/null +++ b/src/rybo/utils/__init__.py @@ -0,0 +1,109 @@ +"""Helper utilities.""" + +import ast +import os +from argparse import Action, ArgumentParser, Namespace +from typing import Any, Optional, Sequence, Union + + +class EnvDefault(Action): + """Custom action used to set CLI arguments via environment variables.""" + + def __init__(self, envvar: str, required=False, default=None, **kwargs) -> None: + """Initialize the action. + + Args: + envvar: Name of the environment variable to read the argument value from. + required: ```True``` to make the argument mandatory, otherwise ```False```. + default: Default value is none is specified by the user. + kwargs: Extra argument configuration options. + """ + if not default and envvar: + default = os.getenv(envvar, None) + + if required and default: + required = False + + super(EnvDefault, self).__init__(default=default, required=required, **kwargs) + + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Union[str, Sequence[Any]]], + option_string: Optional[str] = None, + ) -> None: + """Class setup executed during object instantiation. + + Args: + parser: ArgumentParser that contains the user specified CLI parameters. + namespace: Namespace that contains the argument parser parameters. + values: Parameter value. + option_string: Option strings passed to the parameter. + """ + setattr(namespace, self.dest, values) + + +class EnvFlagDefault(Action): + """Custom action used to set CLI flags via environment variables.""" + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: Optional[bool] = None, + required: bool = False, + help: Optional[str] = None, + envvar: Optional[str] = None, + metavar: Optional[str] = None, + ) -> None: + """Initialize the action. + + If a default value has not been defined, but an environment variable name has + been provided, the action will try to set the flag based on the truthiness of + the provided environment variable (True/False). + + Args: + option_strings: Raw parameter options. + dest: Name of the destination parameter that the parameter value will be + stored under. + default: Default value is none is specified by the user. + required: ```True``` to make the argument mandatory, otherwise ```False```. + help: Help text displayed when the -h/--help parameter is used. + envvar: Name of the environment variable to read the argument value from. + metavar: Name of the metavar used as a placeholder on the help screen. + """ + if not default and envvar: + default = ast.literal_eval(os.getenv(envvar, "False")) + + if required and default: + required = False + + super(EnvFlagDefault, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help, + metavar=metavar, + nargs=0, + ) + self.envvar = envvar + + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Union[str, Sequence[Any]]], + option_string: Optional[str] = None, + ) -> None: + """Class setup executed during object instantiation. + + Args: + parser: ArgumentParser that contains the user specified CLI parameters. + namespace: Namespace that contains the argument parser parameters. + values: Parameter value. + option_string: Option strings passed to the parameter. + """ + setattr(namespace, self.dest, self.const) diff --git a/utils/base64_encoding.py b/src/rybo/utils/base64_encoding.py similarity index 86% rename from utils/base64_encoding.py rename to src/rybo/utils/base64_encoding.py index 0e6848e..99d3a7f 100644 --- a/utils/base64_encoding.py +++ b/src/rybo/utils/base64_encoding.py @@ -1,5 +1,8 @@ """Base64 encoding utilities.""" import base64 +import logging + +logger = logging.getLogger(__name__) def base64_encode_string(message: str) -> None: @@ -12,7 +15,7 @@ def base64_encode_string(message: str) -> None: message_bytes = message.encode("ascii") base64_bytes = base64.b64encode(message_bytes) base64_message = base64_bytes.decode("ascii") - print(base64_message) + logger.info(base64_message) def base64_decode_string(message: str) -> str: @@ -34,5 +37,5 @@ def base64_decode_string(message: str) -> str: # Encoded keywords. Testing the decoding function? message = "cG9ybixzZXgsamVya2luZyBvZmYsc2x1dCxyYXBl" decoded_string = base64_decode_string(message) -print(decoded_string) -print(decoded_string.split(",")) +logger.info(decoded_string) +logger.info(decoded_string.split(",")) diff --git a/utils/common.py b/src/rybo/utils/common.py similarity index 96% rename from utils/common.py rename to src/rybo/utils/common.py index 9384e18..d73c152 100644 --- a/utils/common.py +++ b/src/rybo/utils/common.py @@ -1,8 +1,11 @@ """Common helpers to keep the codebase DRY.""" -from random import SystemRandom -import re +import logging import os +import re from pathlib import Path +from random import SystemRandom + +logger = logging.getLogger(__name__) def random_hex_colour() -> str: @@ -25,8 +28,8 @@ def random_rgb_colour() -> int: """ rnd: SystemRandom = SystemRandom() rbg_colour: int = rnd.choices(range(256), k=3) - print("Generated random rgb colour") - print(rbg_colour) + logger.info("Generated random rgb colour") + logger.info(rbg_colour) return rbg_colour diff --git a/csvmgr.py b/src/rybo/utils/csvmgr.py similarity index 88% rename from csvmgr.py rename to src/rybo/utils/csvmgr.py index 2431c64..12ff96c 100644 --- a/csvmgr.py +++ b/src/rybo/utils/csvmgr.py @@ -1,10 +1,12 @@ """Manage the CSV file that contains metadata about Reddit posts.""" +import logging import os -from typing import Any, Dict +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame -from typing import List + +logger = logging.getLogger(__name__) class CsvWriter: @@ -50,7 +52,9 @@ def is_uploaded(self, id: str) -> bool: csv: DataFrame = pd.read_csv(self.csv_file) # TODO: ??? - results: int = len(csv.loc[(csv["id"] == id) & (csv["uploaded"] == True)]) + results: int = len( + csv.loc[(csv["id"] == id) & (csv["uploaded"] == True)] # noqa: E712 + ) if results > 0: return True else: @@ -72,7 +76,7 @@ def set_uploaded(self, id: str) -> None: if __name__ == "__main__": csvwriter: CsvWriter = CsvWriter() csvwriter.initialise_csv() - print(csvwriter.is_uploaded("snppah")) + logger.info(csvwriter.is_uploaded("snppah")) csvwriter.set_uploaded("snppah") - print(csvwriter.is_uploaded("snppah")) + logger.info(csvwriter.is_uploaded("snppah")) diff --git a/utils/speed_test.py b/src/rybo/utils/speed_test.py similarity index 100% rename from utils/speed_test.py rename to src/rybo/utils/speed_test.py diff --git a/src/rybo/video_generation/__init_-.py b/src/rybo/video_generation/__init_-.py new file mode 100644 index 0000000..e70d862 --- /dev/null +++ b/src/rybo/video_generation/__init_-.py @@ -0,0 +1 @@ +"""Video processors.""" diff --git a/video_generation/video.py b/src/rybo/video_generation/video.py similarity index 75% rename from video_generation/video.py rename to src/rybo/video_generation/video.py index 883d2e2..da54a38 100644 --- a/video_generation/video.py +++ b/src/rybo/video_generation/video.py @@ -7,21 +7,15 @@ import json import logging import os +import sys from os import path from pathlib import Path from random import SystemRandom from typing import Any, Dict, List, Optional -from comments.screenshot import ( - download_screenshot_of_reddit_post_title, - download_screenshots_of_reddit_posts, -) - -import config.settings as settings - -import csvmgr - +import colorama import moviepy.video.fx.all as vfx +from colorama import Fore from moviepy.editor import ( AudioFileClip, ColorClip, @@ -30,24 +24,22 @@ TextClip, VideoFileClip, ) - from praw.models import Comment, Submission from praw.models.comment_forest import CommentForest -import publish.youtube as youtube - -import speech.speech as speech - -from thumbnail.thumbnail import get_font_size - -from utils.common import contains_url, give_emoji_free_text, sanitize_text - -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], +from rybo.comments.screenshot import ( + download_screenshot_of_reddit_post_title, + download_screenshots_of_reddit_posts, ) +from rybo.config import settings +from rybo.publish import youtube +from rybo.speech import speech +from rybo.thumbnail.thumbnail import get_font_size +from rybo.utils import csvmgr +from rybo.utils.common import contains_url, give_emoji_free_text, sanitize_text + +logger = logging.getLogger(__name__) +colorama.init(autoreset=True) def log_group_header(title: str) -> str: @@ -81,13 +73,13 @@ def print_post_details(post: Submission) -> None: Args: post: The Reddit post. """ - logging.info("SubReddit : %s", post.subreddit_name_prefixed) - logging.info("Title : %s", post.title) - logging.info("Score : %s", post.score) - logging.info("ID : %s", post.id) - logging.info("URL : %s", post.url) - logging.info("SelfText : %s", post.selftext) - logging.info("NSFW? : %s", post.over_18) + logger.info("SubReddit : %s", post.subreddit_name_prefixed) + logger.info("Title : %s", post.title) + logger.info("Score : %s", post.score) + logger.info("ID : %s", post.id) + logger.info("URL : %s", post.url) + logger.info("SelfText : %s", post.selftext) + logger.info("NSFW? : %s", post.over_18) def print_comment_details(comment: Comment) -> None: @@ -97,11 +89,11 @@ def print_comment_details(comment: Comment) -> None: comment: The comment. """ if comment.author: - logging.debug("Author : %s", comment.author) - logging.debug("id : %s", comment.id) - logging.debug("Stickied : %s", comment.stickied) - logging.info("Comment : %s", give_emoji_free_text(str(comment.body))) - logging.info("Length : %s", len(comment.body)) + logger.debug("Author : %s", comment.author) + logger.debug("id : %s", comment.id) + logger.debug("Stickied : %s", comment.stickied) + logger.info("Comment : %s", give_emoji_free_text(str(comment.body))) + logger.info("Length : %s", len(comment.body)) class Video: @@ -160,7 +152,7 @@ def get_background(self) -> None: """Select a random background for the video.""" rnd: SystemRandom = SystemRandom() self.background = rnd.choice(seq=os.listdir(settings.background_directory)) - logging.info("Randomly Selecting Background : %s", self.background) + logger.info("Randomly Selecting Background : %s", self.background) def compile(self) -> None: """Compile the video. @@ -189,7 +181,13 @@ def get_random_lines(file_name: Path, num_lines: int) -> str: return "\n".join(random_lines) -def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> None: +def create( + video_directory: Path, + post: Submission, + thumbnails: List[Path], + username: str, + password: str, +) -> None: """Generate a video from a processed reddit post. Args: @@ -197,8 +195,10 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N post: Reddit post that's the main topic of the video. thumbnails: List of images to be embedded in the video. For example, screenshots of user comments. + username: Reddit username. + password: Reddit password. """ - logging.info(log_group_header(title="Processing Reddit Post")) + logger.info(log_group_header(title="Processing Reddit Post")) print_post_details(post) v = Video() @@ -282,12 +282,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N .set_opacity(settings.reddit_comment_opacity) ) if title_clip.w > title_clip.h: - print("Resizing Horizontally") + logger.info("Resizing Horizontally") title_clip = title_clip.resize( width=settings.video_width * settings.reddit_comment_width ) else: - print("Resizing Vertically") + logger.info("Resizing Vertically") title_clip = title_clip.resize(height=settings.video_height * 0.95) else: title_clip = ( @@ -308,14 +308,14 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N newcaster_start: int = t if v.meta.selftext and settings.enable_selftext: - logging.info(log_group_header(title="Processing SelfText")) - logging.info(v.meta.selftext) + logger.info(log_group_header(title="Processing SelfText")) + logger.info(v.meta.selftext) selftext: str = sanitize_text(v.meta.selftext) selftext = give_emoji_free_text(selftext) selftext = os.linesep.join([s for s in selftext.splitlines() if s]) - logging.debug("selftext Length : %s", len(selftext)) + logger.debug("selftext Length : %s", len(selftext)) selftext_lines: List[str] = selftext.splitlines() @@ -327,8 +327,8 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N if selftext_line == " " or selftext_line == " ": continue - logging.debug("selftext length : %s", len(selftext_line)) - logging.debug("selftext_line : %s", selftext_line) + logger.debug("selftext length : %s", len(selftext_line)) + logger.debug("selftext_line : %s", selftext_line) selftext_audio_filepath: str = str( Path(speech_directory, f"selftext_{str(selftext_line_count)}.mp3") ) @@ -336,32 +336,38 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N selftext_audioclip: AudioFileClip = AudioFileClip(selftext_audio_filepath) current_clip_text += f"{selftext_line}\n" - logging.debug("Current Clip Text :") - logging.debug(current_clip_text) - logging.debug("SelfText Fontsize : %s", settings.text_fontsize) - - selftext_clip: TextClip = ( - TextClip( - current_clip_text, - font=settings.text_font, - fontsize=settings.text_fontsize, - color=settings.text_color, - size=txt_clip_size, - kerning=-1, - method="caption", - # bg_color=settings.text_bg_color, - align="West", + logger.debug("Current Clip Text :") + logger.debug(current_clip_text) + logger.debug("SelfText Fontsize : %s", settings.text_fontsize) + + try: + selftext_clip: TextClip = ( + TextClip( + current_clip_text, + font=settings.text_font, + fontsize=settings.text_fontsize, + color=settings.text_color, + size=txt_clip_size, + kerning=-1, + method="caption", + # bg_color=settings.text_bg_color, + align="West", + ) + .set_position((clip_margin, clip_margin_top)) + .set_duration(selftext_audioclip.duration + settings.pause) + .set_audio(selftext_audioclip) + .set_start(t) + .set_opacity(settings.text_bg_opacity) + .volumex(1.5) ) - .set_position((clip_margin, clip_margin_top)) - .set_duration(selftext_audioclip.duration + settings.pause) - .set_audio(selftext_audioclip) - .set_start(t) - .set_opacity(settings.text_bg_opacity) - .volumex(1.5) - ) + except IOError as ioerr: + logger.exception( + f"{Fore.RED}An unexpected error has occured.", exc_info=ioerr + ) + sys.exit(1) if selftext_clip.h > settings.video_height: - logging.debug("Text exceeded Video Height, reset text") + logger.debug("Text exceeded Video Height, reset text") current_clip_text = f"{selftext_line}\n" selftext_clip = ( TextClip( @@ -383,18 +389,18 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if selftext_clip.h > settings.video_height: - logging.debug("Comment Text Too Long, Skipping Comment") + logger.debug("Comment Text Too Long, Skipping Comment") continue t += selftext_audioclip.duration + settings.pause v.duration += selftext_audioclip.duration + settings.pause v.clips.append(selftext_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) - logging.info(log_group_header(title="Finished Processing SelfText")) + logger.info("Current Video Duration : %s", v.duration) + logger.info(log_group_header(title="Finished Processing SelfText")) static_clip: VideoFileClip = ( VideoFileClip("static.mp4") @@ -419,16 +425,16 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N rejected_comments: List[Comment] = [] - logging.info(log_group_header(title="Filtering Reddit Comments")) + logger.info(log_group_header(title="Filtering Reddit Comments")) for count, c in enumerate(all_comments): - logging.info(log_group_subheader(title=f"Comment # {str(count)}")) + logger.info(log_group_subheader(title=f"Comment # {str(count)}")) print_comment_details(c) comment: str = c.body if len(comment) > settings.comment_length_max: - logging.info( + logger.info( "Status : REJECTED, Comment exceeds max character length : %s", settings.comment_length_max, ) @@ -436,12 +442,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N continue if comment == "[removed]" or comment == "[deleted]": - logging.info("Status : REJECTED, Skipping Comment : %s", comment) + logger.info("Status : REJECTED, Skipping Comment : %s", comment) rejected_comments.append(c) continue if "covid" in comment.lower() or "vaccine" in comment.lower(): - logging.info( + logger.info( "Status : REJECTED, Covid related, \ Youtube will Channel Strike..: %s", comment, @@ -452,24 +458,24 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N comment = give_emoji_free_text(comment) comment = os.linesep.join([s for s in comment.splitlines() if s]) - logging.debug("Comment Length : %s", len(comment)) + logger.debug("Comment Length : %s", len(comment)) if c.stickied: - logging.info("Status : REJECTED, Skipping Stickied Comment...") + logger.info("Status : REJECTED, Skipping Stickied Comment...") rejected_comments.append(c) continue if contains_url(comment): - logging.info("Status : REJECTED, Skipping Comment with URL in it...") + logger.info("Status : REJECTED, Skipping Comment with URL in it...") rejected_comments.append(c) continue - logging.info("Status : ACCEPTED") + logger.info("Status : ACCEPTED") accepted_comments.append(c) if len(accepted_comments) == settings.comment_limit: - logging.info("Rejected Comments : %s", len(rejected_comments)) - logging.info("Accepted Comments : %s", len(accepted_comments)) + logger.info("Rejected Comments : %s", len(rejected_comments)) + logger.info("Accepted Comments : %s", len(accepted_comments)) break screenshot_directory = Path(settings.screenshot_directory, v.meta.id) if settings.commentstyle == "reddit": @@ -477,10 +483,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N accepted_comments, f"http://reddit.com{v.meta.permalink}", screenshot_directory, + username, + password, ) for count, accepted_comment in enumerate(accepted_comments): - logging.info( + logger.info( "=== Processing Reddit Comment %s/%s ===", count, len(accepted_comments) ) @@ -509,22 +517,22 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) ) except Exception as e: - print(e) + logger.info(e) continue else: - logging.info("Comment image not found : %s", img_path) + logger.info("Comment image not found : %s", img_path) continue if img_clip.h > settings.video_height: - logging.info("Comment larger than video height : %s", img_path) + logger.info("Comment larger than video height : %s", img_path) continue if v.duration + audioclip.duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length ) - logging.info("Used %s/%s comments", count, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", count, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break t += audioclip.duration + settings.pause @@ -532,23 +540,23 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N v.clips.append(img_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) + logger.info("Current Video Duration : %s", v.duration) if settings.commentstyle == "text": comment_lines: List[str] = accepted_comment.body.splitlines() for ccount, comment_line in enumerate(comment_lines): if comment_line == "​": - logging.info("Skip zero space character comment : %s", comment) + logger.info("Skip zero space character comment : %s", comment) continue if comment_line == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue - logging.debug("comment_line : %s", comment_line) + logger.debug("comment_line : %s", comment_line) audio_filepath = str( Path( speech_directory, @@ -559,8 +567,8 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N audioclip = AudioFileClip(audio_filepath) current_clip_text += f"{comment_line}\n\n" - logging.debug("Current Clip Text :") - logging.debug(current_clip_text) + logger.debug("Current Clip Text :") + logger.debug(current_clip_text) txt_clip: TextClip = ( TextClip( @@ -583,7 +591,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if txt_clip.h > settings.video_height: - logging.debug("Text exceeded Video Height, reset text") + logger.debug("Text exceeded Video Height, reset text") current_clip_text = f"{comment_line}\n\n" txt_clip = ( TextClip( @@ -605,58 +613,58 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if txt_clip.h > settings.video_height: - logging.debug("Comment Text Too Long, Skipping Comment") + logger.debug("Comment Text Too Long, Skipping Comment") continue total_duration: int = v.duration + audioclip.duration if total_duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length, ) - logging.info( + logger.info( "Used %s/%s comments", ccount, len(accepted_comments), ) - logging.info("=== Finished Processing Comments ===") + logger.info("=== Finished Processing Comments ===") break t += audioclip.duration + settings.pause v.duration += audioclip.duration + settings.pause v.clips.append(txt_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) + logger.info("Current Video Duration : %s", v.duration) if v.duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length ) - logging.info("Used %s/%s comments", ccount, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", ccount, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break if count == settings.comment_limit: - logging.info( + logger.info( "Reached Maximum Number of Comments Limit : %s", settings.comment_limit, ) - logging.info("Used %s/%s comments", ccount, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", ccount, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break else: - logging.info("Skipping comments!") + logger.info("Skipping comments!") - logging.info(log_group_subheader(title="Adding Background Clip")) + logger.info(log_group_subheader(title="Adding Background Clip")) if settings.enable_background: background_filepath: Path = Path( settings.background_directory, str(v.background) ) - logging.info("Background : %s", background_filepath) + logger.info("Background : %s", background_filepath) background_clip: VideoFileClip = ( VideoFileClip(background_filepath) @@ -666,24 +674,24 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if settings.orientation == "portrait": - print("Portrait mode, cropping and resizing!") + logger.info("Portrait mode, cropping and resizing!") background_clip = background_clip.crop( x1=1166.6, y1=0, x2=2246.6, y2=1920 ).resize((settings.vertical_video_width, settings.vertical_video_height)) if background_clip.duration < v.duration: - logging.debug("Looping Background") + logger.debug("Looping Background") # background_clip = vfx.make_loopable(background_clip, cross=0) background_clip = vfx.loop( background_clip, duration=v.duration ).without_audio() video_duration: str = str(background_clip.duration) - logging.debug("Looped Background Clip Duration : %s", video_duration) + logger.debug("Looped Background Clip Duration : %s", video_duration) else: - logging.debug("Not Looping Background") + logger.debug("Not Looping Background") background_clip = background_clip.set_duration(v.duration) else: - logging.info("Background not enabled...") + logger.info("Background not enabled...") background_clip = ColorClip( size=(settings.video_width, settings.video_height), color=settings.background_colour, @@ -692,7 +700,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N v.clips.insert(0, background_clip) if settings.enable_overlay: - logging.info(log_group_subheader(title="Adding Overlay Clip")) + logger.info(log_group_subheader(title="Adding Overlay Clip")) clip_video_overlay: VideoFileClip = ( VideoFileClip(settings.video_overlay_filepath) .set_start(tb) @@ -702,22 +710,22 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if clip_video_overlay.duration < v.duration: - logging.debug("Looping Overlay") + logger.debug("Looping Overlay") # background_clip = vfx.make_loopable(background_clip, cross=0) clip_video_overlay = vfx.loop( clip_video_overlay, duration=v.duration ).without_audio() video_duration = str(clip_video_overlay.duration) - logging.debug("Looped Overlay Clip Duration : %s", video_duration) + logger.debug("Looped Overlay Clip Duration : %s", video_duration) else: - logging.debug("Not Looping Overlay") + logger.debug("Not Looping Overlay") clip_video_overlay = clip_video_overlay.set_duration(v.duration) v.clips.insert(1, clip_video_overlay) if settings.enable_newscaster and settings.newscaster_filepath: - logging.info(log_group_subheader(title="Adding Newcaster Clip")) - logging.info("Newscaster File Path: %s", settings.newscaster_filepath) + logger.info(log_group_subheader(title="Adding Newcaster Clip")) + logger.info("Newscaster File Path: %s", settings.newscaster_filepath) clip_video_newscaster: VideoFileClip = ( VideoFileClip(settings.newscaster_filepath) .set_position(settings.newscaster_position) @@ -728,7 +736,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if settings.newscaster_remove_greenscreen: - logging.info(log_group_subheader(title="Removing Newcaster Green Screen")) + logger.info(log_group_subheader(title="Removing Newcaster Green Screen")) # Green Screen Video https://github.com/Zulko/moviepy/issues/964 clip_video_newscaster = clip_video_newscaster.fx( vfx.mask_color, @@ -738,15 +746,15 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if clip_video_newscaster.duration < v.duration: - logging.debug("Looping Newscaster") + logger.debug("Looping Newscaster") clip_video_newscaster = vfx.loop( clip_video_newscaster, duration=v.duration - newcaster_start ).without_audio() - logging.debug( + logger.debug( "Looped Newscaster Clip Duration : %s", clip_video_newscaster.duration ) else: - logging.debug("Not Looping Newscaster") + logger.debug("Not Looping Newscaster") clip_video_newscaster = clip_video_newscaster.set_duration( v.duration - newcaster_start ) @@ -786,31 +794,31 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N csvwriter.write_entry(row=row) if settings.enable_compilation: - logging.info(log_group_subheader(title="Compiling Video Clip")) - logging.info("Compiling video, this takes a while, please be patient : ") + logger.info(log_group_subheader(title="Compiling Video Clip")) + logger.info("Compiling video, this takes a while, please be patient : ") post_video.write_videofile(v.filepath, fps=24) else: - logging.info("Skipping Video Compilation --enable_compilation passed") + logger.info("Skipping Video Compilation --enable_compilation passed") if settings.enable_compilation and settings.enable_upload: if path.exists("client_secret.json") and path.exists("credentials.storage"): if csvwriter.is_uploaded(v.meta.id): - logging.info("Already uploaded according to data.csv") + logger.info("Already uploaded according to data.csv") else: - logging.info( + logger.info( log_group_subheader(title="Uploading Video Clip to YouTube") ) try: youtube.publish(v) except Exception as e: - print(e) + logger.info(e) else: csvwriter.set_uploaded(v.meta.id) else: - logging.info( + logger.info( "Skipping upload, missing either \ client_secret.json or credentials.storage file." ) else: - logging.info("Skipping Upload...") + logger.info("Skipping Upload...")