Cmscom/hawthorn.master from ginkgo

CHANGELOG.md

@@ -232,6 +232,12 @@
   - Added `NGINX_EDXAPP_CMS_APP_EXTRA`, which makes it possible to add custom settings to the site configuration for Studio.
   - Added `NGINX_EDXAPP_LMS_APP_EXTRA`, which makes it possible to add custom settings to the site configuration for the LMS.
 
+- Role: edxapp
+  - Added creation of enterprise_worker user to provisioning. This user is used by the edx-enterprise package when making API requests to Open edX IDAs.
+
+- Role: forum
+  - Added `FORUM_REBUILD_INDEX` to rebuild the ElasticSearch index from the database, when enabled.  Default: `False`.
+
 - Role: edxapp
   - Let `confirm_email` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`.
   - Let `terms_of_service` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`.

README.rst

@@ -1,6 +1,12 @@
 Configuration Management
 ########################
 
+About
+**********
+
+fro CMScom config
+
+
 Introduction
 ************
 

playbooks/analytics_single_cmscom.yml

@@ -0,0 +1,22 @@
+# for insights
+- name: Deploy all analytics services to a single node
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    migrate_db: "no"
+    disable_edx_services: false
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: False
+  roles:
+#    - aws
+#    - mysql
+#    - edxlocal
+    - memcache
+    - analytics_api
+    - analytics_pipeline
+    - role: nginx
+      nginx_sites:
+        - insights
+    - insights

playbooks/create_mongo_users.yml

@@ -28,14 +28,14 @@
     ansible_python_interpreter: "/usr/bin/env python"
   tasks:
     - name: install python mongo module
-      pip: name=pymongo state=present
+      pip: name=pymongo state=present version="{{ pymongo_version }}"
 
     - name: create a mongodb user
       mongodb_user:
         database: "{{ item.database }}"
         login_user: "{{ MONGO_ADMIN_USER }}"
         login_password: "{{ MONGO_ADMIN_PASSWORD }}"
-        login_host: "{{ login_host }}"
+        login_host: "{{ MONGO_IP }}"
         name: "{{ item.user }}"
         password: "{{ item.password }}"
         roles: "{{ item.roles }}"

playbooks/edx_cmscom.yml

@@ -0,0 +1,101 @@
+---
+
+# Example sandbox configuration
+# for single server community
+# installs
+
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+# TODO: 2019-07-19
+# added mongo setting
+#- include: edx-east/create_mongo_users.yml
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    migrate_db: "yes"
+    EDXAPP_LMS_NGINX_PORT: '80'
+    edx_platform_version: 'master'
+    # Set to false if deployed behind another proxy/load balancer.
+    NGINX_SET_X_FORWARDED_HEADERS: True
+    DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}'
+    ecommerce_create_demo_data: false
+    credentials_create_demo_data: false
+    SANDBOX_ENABLE_DISCOVERY: true
+    SANDBOX_ENABLE_ECOMMERCE: false
+    SANDBOX_ENABLE_ANALYTICS_API: true
+    SANDBOX_ENABLE_INSIGHTS: false
+    SANDBOX_ENABLE_RABBITMQ: true
+    JOURNALS_ENABLED: false
+    # test 20191119
+    EDXAPP_MYSQL_HOST: localhost
+  roles:
+    - role: swapfile
+      SWAPFILE_SIZE: 4GB
+    - role: nginx
+      nginx_sites:
+      - certs
+      - cms
+      - lms
+      - forum
+      - xqueue
+      nginx_default_sites:
+      - lms
+# test 20191119 off to on
+    - role: edxlocal
+      when: EDXAPP_MYSQL_HOST == 'localhost'
+#    - role: edxremote
+    - role: memcache
+      when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)"
+# 20191220 comment in for local install
+#    - role: mongo_3_2
+#      when: "'localhost' in EDXAPP_MONGO_HOSTS"
+    - role: mongo_3_2
+      when: "'localhost' in EDXAPP_MONGO_HOSTS"
+    - role: rabbitmq
+      rabbitmq_ip: 127.0.0.1
+      when: SANDBOX_ENABLE_RABBITMQ
+    - role: edxapp
+      celery_worker: True
+    - edxapp
+    - role: ecommerce
+      when: SANDBOX_ENABLE_ECOMMERCE
+    - role: ecomworker
+      ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
+      when: SANDBOX_ENABLE_ECOMMERCE
+    - role: analytics_api
+      when: SANDBOX_ENABLE_ANALYTICS_API
+    - role: insights
+      when: SANDBOX_ENABLE_INSIGHTS
+    # not ready yet: - edx_notes_api
+#    - demo
+    - oauth_client_setup
+    - oraclejdk
+    - role: elasticsearch
+      when: "'localhost' in EDXAPP_ELASTIC_SEARCH_CONFIG|map(attribute='host')"
+    - forum
+    - role: discovery
+      when: SANDBOX_ENABLE_DISCOVERY
+    - role: journals
+      when: JOURNALS_ENABLED
+    - role: notifier
+      NOTIFIER_DIGEST_TASK_INTERVAL: 5
+    - role: xqueue
+      update_users: True
+    - certs
+    - edx_ansible
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: postfix_queue
+      when: POSTFIX_QUEUE_EXTERNAL_SMTP_HOST != ''
+    - role: datadog-uninstall
+      when: not COMMON_ENABLE_DATADOG

playbooks/edx_sandbox.yml

@@ -40,6 +40,7 @@
       - lms
       - forum
       - xqueue
+      - ecommerce
       nginx_default_sites:
       - lms
     - role: edxlocal
@@ -54,8 +55,7 @@
     - role: edxapp
       celery_worker: True
     - edxapp
-    - role: ecommerce
-      when: SANDBOX_ENABLE_ECOMMERCE
+    - ecommerce
     - role: ecomworker
       ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
       when: SANDBOX_ENABLE_ECOMMERCE

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -220,15 +220,17 @@
     - install
     - install:configuration
 
-- name: Grant access to table storing test data in output database
-  mysql_user:
-    user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.username }}"
-    password: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.password }}"
-    priv: 'acceptance%.*:ALL'
-    append_privs: yes
-  tags:
-    - install
-    - install:configuration
+# TODO: すでに設定されているデータ。おそらく不要なはず
+#- name: Grant access to table storing test data in output database
+#  mysql_user:
+#    user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.username }}"
+#    password: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.password }}"
+#    host: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.host }}"
+#    priv: 'acceptance%.*:ALL'
+#    append_privs: yes
+#  tags:
+#    - install
+#    - install:configuration
 
 - name: Test if Hive metadata store schema exists
   shell: ". {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && {{ HIVE_HOME }}/bin/hive | tr '\n' ' '"

playbooks/roles/edxremote/defaults/main.yml

@@ -0,0 +1,63 @@
+---
+edxlocal_debian_pkgs:
+  - postfix
+  - libjpeg-dev
+
+edxlocal_databases:
+  - "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ INSIGHTS_DATABASE_NAME | default(None) }}"
+  - "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}"
+  - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}"
+  - "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}"
+  - "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}"
+  - "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}"
+  - "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}"
+
+edxlocal_database_users:
+  - {
+      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
+      pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
+      user: "{{ INSIGHTS_DATABASE_USER | default(None) }}",
+      pass: "{{ INSIGHTS_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
+      user: "{{ XQUEUE_MYSQL_USER | default(None) }}",
+      pass: "{{ XQUEUE_MYSQL_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}",
+      user: "{{ EDXAPP_MYSQL_USER | default(None) }}",
+      pass: "{{ EDXAPP_MYSQL_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}",
+      user: "{{ EDXAPP_MYSQL_CSMH_USER | default(None) }}",
+      pass: "{{ EDXAPP_MYSQL_CSMH_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_NAME | default(None) }}",
+      user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_USER | default(None) }}",
+      pass: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
+      user: "{{ HIVE_METASTORE_DATABASE_USER | default(None) }}",
+      pass: "{{ HIVE_METASTORE_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ CREDENTIALS_MYSQL_USER | default(None) }}",
+      pass: "{{ CREDENTIALS_MYSQL_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ DISCOVERY_MYSQL_USER | default(None) }}",
+      pass: "{{ DISCOVERY_MYSQL_PASSWORD | default(None) }}"
+    }

playbooks/roles/edxremote/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - common
+#  - mysql

playbooks/roles/edxremote/tasks/main.yml

@@ -0,0 +1,109 @@
+          # Installs packages to run edx locally on a single instance
+#
+---
+- name: Install packages needed for single server
+  apt:
+    name: "{{ item }}"
+    install_recommends: yes
+    state: present
+  with_items: "{{ edxlocal_debian_pkgs }}"
+
+# TODO: Add a test to make sure mysql is running.
+
+- name: create databases
+  mysql_db:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    db: "{{ item }}"
+    state: present
+    encoding: utf8
+  when: item != None and item != ''
+  with_items: "{{ edxlocal_databases }}"
+
+- name: create database users
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: "{{ item.user }}"
+    password: "{{ item.pass }}"
+    priv: "{{ item.db }}.*:ALL"
+    append_privs: yes
+  when: item.db != None and item.db != ''
+  with_items: "{{ edxlocal_database_users }}"
+
+- name: setup the migration db user
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+    password: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
+    priv: "{{ item }}.*:ALL"
+    append_privs: yes
+  when: item != None and item != ''
+  with_items: "{{ edxlocal_databases }}"
+
+- name: create api user for the analytics api
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: "api001"
+    password: "{{ ANALYTICS_API_DATABASES.default.PASSWORD }}"
+    priv: '{{ ANALYTICS_API_DATABASES.default.NAME }}.*:ALL/reports.*:SELECT'
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
+
+- name: create read-only reports user for the analytics-api
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: reports001
+    password: "{{ ANALYTICS_API_DATABASES.reports.PASSWORD }}"
+    priv: '{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:SELECT'
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
+
+- name: create a database for the hive metastore
+  mysql_db:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    db: "{{ HIVE_METASTORE_DATABASE.name }}"
+    state: "present"
+    encoding: "latin1"
+  when: HIVE_METASTORE_DATABASE is defined
+
+- name: setup the edx-notes-api db user
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: "{{ EDX_NOTES_API_MYSQL_DB_USER }}"
+    password: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}"
+    priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE"
+  when: EDX_NOTES_API_MYSQL_DB_USER is defined
+
+- name: setup the read-only db user
+  mysql_user:
+    login_host: "{{ EDXAPP_MYSQL_HOST }}"
+    login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    host: '%'
+    name: "{{ COMMON_MYSQL_READ_ONLY_USER }}"
+    password: "{{ COMMON_MYSQL_READ_ONLY_PASS }}"
+    priv: "*.*:ALL"
+
+- name: setup the admin db user
+  mysql_user:
+    name: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    priv: "*.*:CREATE USER"
+  when: EDXAPP_MYSQL_HOST == 'localhost'
+