From cb4fba6665a674082f344c2128b31b8b715648e5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 7 Jul 2017 14:19:05 -0400 Subject: [PATCH 01/60] Pin the discovery version to the release we want (cherry picked from commit b47e7745c7be57305c95197fcf6c21bd514e2491) --- playbooks/vagrant-devstack.yml | 1 + vagrant/base/devstack/Vagrantfile | 2 ++ vagrant/base/fullstack/Vagrantfile | 1 + vagrant/release/devstack/Vagrantfile | 2 ++ 4 files changed, 6 insertions(+) diff --git a/playbooks/vagrant-devstack.yml b/playbooks/vagrant-devstack.yml index 4747899c9e2..c3b63bafffb 100644 --- a/playbooks/vagrant-devstack.yml +++ b/playbooks/vagrant-devstack.yml @@ -5,6 +5,7 @@ vars: migrate_db: 'yes' devstack: true + edx_django_service_is_devstack: true disable_edx_services: true mongo_enable_journal: false EDXAPP_NO_PREREQ_INSTALL: 0 diff --git a/vagrant/base/devstack/Vagrantfile b/vagrant/base/devstack/Vagrantfile index cd95cb6fe61..0630aa88874 100644 --- a/vagrant/base/devstack/Vagrantfile +++ b/vagrant/base/devstack/Vagrantfile @@ -26,6 +26,7 @@ VERSION_VARS = [ 'NOTIFIER_VERSION', 'ECOMMERCE_VERSION', 'ECOMMERCE_WORKER_VERSION', + 'DISCOVERY_VERSION', ] Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| @@ -51,6 +52,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8110, host: 8110 # Insights config.vm.network :forwarded_port, guest: 50070, host: 50070 # HDFS Admin UI config.vm.network :forwarded_port, guest: 8088, host: 8088 # Hadoop Resource Manager + config.vm.network :forwarded_port, guest: 18381, host: 18381 # Course discovery end config.ssh.insert_key = true diff --git a/vagrant/base/fullstack/Vagrantfile b/vagrant/base/fullstack/Vagrantfile index 920812ae79f..ce63776d3ed 100644 --- a/vagrant/base/fullstack/Vagrantfile +++ b/vagrant/base/fullstack/Vagrantfile @@ -23,6 +23,7 @@ VERSION_VARS = [ 'NOTIFIER_VERSION', 'ECOMMERCE_VERSION', 'ECOMMERCE_WORKER_VERSION', + 'DISCOVERY_VERSION', ] Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index c4cd01c3504..a00392c751b 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -21,6 +21,7 @@ VERSION_VARS = [ 'NOTIFIER_VERSION', 'ECOMMERCE_VERSION', 'ECOMMERCE_WORKER_VERSION', + 'DISCOVERY_VERSION', ] MOUNT_DIRS = { @@ -134,6 +135,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 9876, host: 9876 # ORA2 Karma tests config.vm.network :forwarded_port, guest: 50070, host: 50070 # HDFS Admin UI config.vm.network :forwarded_port, guest: 8088, host: 8088 # Hadoop Resource Manager + config.vm.network :forwarded_port, guest: 18381, host: 18381 # Course discovery end config.ssh.insert_key = true From 5a3b2d18f483438cb0bf097e6f77544e4913c58b Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Thu, 13 Jul 2017 10:49:11 -0400 Subject: [PATCH 02/60] Copy pattern for delegating auth to the api, skipping basic auth, from the LMS to CMS (cherry picked from commit 43d85953761292c527c04937b75f19c44734673b) --- .../nginx/templates/edx/app/nginx/sites-available/cms.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 index 8d19daa5e55..ef2fb9ced1a 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 @@ -123,6 +123,13 @@ error_page {{ k }} {{ v }}; try_files $uri @proxy_to_cms_app; } + # The api is accessed using OAUTH2 which + # uses the authorization header so we can't have + # basic auth on it as well. + location /api { + try_files $uri @proxy_to_cms_app; + } + {% include "robots.j2" %} {% include "static-files.j2" %} From eda65fd2d7bb1458b8eaa751b39070d5453bbddf Mon Sep 17 00:00:00 2001 From: Ari Rizzitano Date: Tue, 11 Jul 2017 18:03:56 -0400 Subject: [PATCH 03/60] upgrade node version to 6.11.1 (cherry picked from commit 9cd474062d1ff23737662fbc955db79051e7db83) --- playbooks/roles/common_vars/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 9d63da35951..4d066923987 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -99,7 +99,7 @@ COMMON_ENABLE_NEWRELIC_APP: False COMMON_ENABLE_MINOS: False COMMON_TAG_EC2_INSTANCE: False common_boto_version: '2.34.0' -common_node_version: '6.9.4' +common_node_version: '6.11.1' common_redhat_pkgs: - ntp - lynx From 905498573d711a5f048bd988cbbc03a58744197e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 14 Jul 2017 17:14:51 -0400 Subject: [PATCH 04/60] Vagrantfile updates for ginkgo, and ficus changes that never made it to master --- vagrant/release/devstack/Vagrantfile | 7 ++++++- vagrant/release/fullstack/Vagrantfile | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index a00392c751b..49bfc8c7691 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -43,11 +43,16 @@ end # to a name and a file path, which are used for retrieving # a Vagrant box from the internet. openedx_releases = { + "open-release/ginkgo.master" => "ginkgo-devstack-2017-07-14", + "open-release/ginkgo.1rc1" => "ginkgo-devstack-2017-07-14", + "open-release/ficus.master" => "ficus-devstack-2017-02-07", "open-release/ficus.1rc1" => "ficus-devstack-2017-01-11", "open-release/ficus.1rc3" => "ficus-devstack-2017-02-07", "open-release/ficus.1rc4" => "ficus-devstack-2017-02-07", "open-release/ficus.1" => "ficus-devstack-2017-02-07", + "open-release/ficus.2" => "ficus-devstack-2017-02-07", + "open-release/ficus.3" => "ficus-devstack-2017-02-07", "open-release/eucalyptus.master" => "eucalyptus-devstack-2016-09-01", "open-release/eucalyptus.1rc2" => "eucalyptus-devstack-2016-08-19", @@ -64,7 +69,7 @@ openedx_releases = { # Cypress is deprecated and unsupported # Birch is deprecated and unsupported } -openedx_releases.default = "devstack-periodic-2017-06-19" +openedx_releases.default = "ficus-devstack-2017-02-07" openedx_release = ENV['OPENEDX_RELEASE'] boxname = ENV['OPENEDX_BOXNAME'] diff --git a/vagrant/release/fullstack/Vagrantfile b/vagrant/release/fullstack/Vagrantfile index ecb71404324..a9241c3c8ee 100644 --- a/vagrant/release/fullstack/Vagrantfile +++ b/vagrant/release/fullstack/Vagrantfile @@ -2,16 +2,20 @@ Vagrant.require_version ">= 1.8.7" VAGRANTFILE_API_VERSION = "2" -MEMORY = 4096 +MEMORY = 6144 CPU_COUNT = 2 # map the name of the git branch that we use for a release # to a name and a file path, which are used for retrieving # a Vagrant box from the internet. openedx_releases = { + "open-release/ginkgo.1rc1" => "ginkgo-fullstack-2017-07-14", + "open-release/ficus.1rc3" => "ficus-fullstack-2017-02-07", "open-release/ficus.1rc4" => "ficus-fullstack-2017-02-15", "open-release/ficus.1" => "ficus-fullstack-2017-02-15", + "open-release/ficus.2" => "ficus-fullstack-2017-03-28", + "open-release/ficus.3" => "ficus-fullstack-2017-04-20", "open-release/eucalyptus/1rc1" => "eucalyptus-fullstack-1rc1", "open-release/eucalyptus.1rc2" => "eucalyptus-fullstack-2016-08-19", @@ -36,7 +40,7 @@ openedx_releases = { # Cypress is deprecated and unsupported # Birch is deprecated and unsupported } -openedx_releases.default = "ficus-fullstack-2017-02-15" +openedx_releases.default = "ficus-fullstack-2017-04-20" openedx_release = ENV['OPENEDX_RELEASE'] boxname = ENV['OPENEDX_BOXNAME'] From df9b5753d14e5dff91522b55ce3520579abcf9c9 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 15 Jul 2017 09:10:23 -0400 Subject: [PATCH 05/60] Fullstack memory bumps from ficus that never made it to master. (cherry picked from commit a6c0f2a58af6c432069a9e431c05c7a08e64b1cc) --- vagrant/base/fullstack/Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vagrant/base/fullstack/Vagrantfile b/vagrant/base/fullstack/Vagrantfile index ce63776d3ed..3da90d2c91e 100644 --- a/vagrant/base/fullstack/Vagrantfile +++ b/vagrant/base/fullstack/Vagrantfile @@ -2,7 +2,7 @@ Vagrant.require_version ">= 1.8.7" VAGRANTFILE_API_VERSION = "2" -MEMORY = 4096 +MEMORY = 6144 CPU_COUNT = 2 vm_guest_ip = "192.168.33.10" From bb5bc41ffb4328624103b070cc92d24d8f390f0c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 17 Jul 2017 11:52:54 -0400 Subject: [PATCH 06/60] Have to pin analytics code to the right version --- vagrant/base/fullstack/Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vagrant/base/fullstack/Vagrantfile b/vagrant/base/fullstack/Vagrantfile index 3da90d2c91e..6f4714fb935 100644 --- a/vagrant/base/fullstack/Vagrantfile +++ b/vagrant/base/fullstack/Vagrantfile @@ -21,6 +21,8 @@ VERSION_VARS = [ 'xqueue_version', 'demo_version', 'NOTIFIER_VERSION', + 'INSIGHTS_VERSION', + 'ANALYTICS_API_VERSION', 'ECOMMERCE_VERSION', 'ECOMMERCE_WORKER_VERSION', 'DISCOVERY_VERSION', From 77f39b9ae70de47fea3e3827b0a6a0fa7efa39ef Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 17 Jul 2017 16:32:56 -0400 Subject: [PATCH 07/60] No more need for SANDBOX_ENABLE_ECOMMERCE (cherry picked from commit 3f496fb1e3ddf0ea8c0fea46eee5996f1ee0123c) --- playbooks/edx_sandbox.yml | 13 ++----------- util/install/sandbox.sh | 1 - 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/playbooks/edx_sandbox.yml b/playbooks/edx_sandbox.yml index 41588295aab..8da5b3025af 100644 --- a/playbooks/edx_sandbox.yml +++ b/playbooks/edx_sandbox.yml @@ -21,10 +21,6 @@ edx_platform_version: 'master' # Set to false if deployed behind another proxy/load balancer. NGINX_SET_X_FORWARDED_HEADERS: True - # These should stay false for the public AMI - COMMON_ENABLE_DATADOG: False - SANDBOX_ENABLE_ECOMMERCE: False - COMMON_ENABLE_SPLUNKFORWARDER: False DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}' roles: - role: swapfile @@ -36,12 +32,9 @@ - lms - forum - xqueue + - ecommerce nginx_default_sites: - lms - - role: nginx - nginx_sites: - - ecommerce - when: SANDBOX_ENABLE_ECOMMERCE - role: edxlocal when: EDXAPP_MYSQL_HOST == 'localhost' - role: memcache @@ -53,11 +46,9 @@ - role: edxapp celery_worker: True - edxapp - - role: ecommerce - when: SANDBOX_ENABLE_ECOMMERCE + - ecommerce - role: ecomworker ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 - when: SANDBOX_ENABLE_ECOMMERCE - analytics_api - insights # not ready yet: - edx_notes_api diff --git a/util/install/sandbox.sh b/util/install/sandbox.sh index 0a09566efff..04915db9c5d 100644 --- a/util/install/sandbox.sh +++ b/util/install/sandbox.sh @@ -54,7 +54,6 @@ VERSION_VARS=( ECOMMERCE_WORKER_VERSION ) -EXTRA_VARS="-e SANDBOX_ENABLE_ECOMMERCE=True $EXTRA_VARS" for var in ${VERSION_VARS[@]}; do # Each variable can be overridden by a similarly-named environment variable, # or OPENEDX_RELEASE, if provided. From 4e839bb38cf8e3c1cfb0c049a7ee443d9666f862 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Fri, 14 Jul 2017 16:46:12 +0930 Subject: [PATCH 08/60] Add optional task to rebuild the forum elasticsearch index set FORUM_REBUILD_INDEX to run the rebuild_index rake task. (cherry picked from commit d07e6fca7cdb4222ed1f40dd649b310866ad611f) --- CHANGELOG.md | 3 +++ playbooks/roles/forum/defaults/main.yml | 3 +++ playbooks/roles/forum/tasks/deploy.yml | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f52fbf5ba7..ed23a7921f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +- Role: forum + - Added `FORUM_REBUILD_INDEX` to rebuild the ElasticSearch index from the database, when enabled. Default: `False`. + - Role: edxapp - Let `confirm_email` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`. - Let `terms_of_service` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`. diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 2d458081713..bcda429c6bf 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -45,6 +45,9 @@ FORUM_USE_TCP: false # wait this long before attempting to restart it FORUM_RESTART_DELAY: 60 +# Set to rebuild the forum ElasticSearch index from the database. +FORUM_REBUILD_INDEX: false + forum_environment: RBENV_ROOT: "{{ forum_rbenv_root }}" GEM_HOME: "{{ forum_gem_root }}" diff --git a/playbooks/roles/forum/tasks/deploy.yml b/playbooks/roles/forum/tasks/deploy.yml index fd8c27c0f59..a70bf88f8a9 100644 --- a/playbooks/roles/forum/tasks/deploy.yml +++ b/playbooks/roles/forum/tasks/deploy.yml @@ -71,6 +71,17 @@ - migrate - migrate:db +- name: rebuild elasticsearch indexes + command: "{{ forum_code_dir }}/bin/rake search:rebuild_index" + args: + chdir: "{{ forum_code_dir }}" + become_user: "{{ forum_user }}" + environment: "{{ forum_environment }}" + when: migrate_db is defined and migrate_db|lower == "yes" and FORUM_REBUILD_INDEX|bool + tags: + - migrate + - migrate:db + # call supervisorctl update. this reloads # the supervisorctl config and restarts # the services if any of the configurations From 9da256a7223584d62bc8464e438fc12076bf6be1 Mon Sep 17 00:00:00 2001 From: Douglas Hall Date: Fri, 28 Jul 2017 16:41:24 -0400 Subject: [PATCH 09/60] Revert "Revert "[asadiqbal08/ENT-322] Add 'enterprise_worker' user to LMS user provisioning"" --- CHANGELOG.md | 3 +++ playbooks/roles/common_vars/defaults/main.yml | 3 +++ playbooks/roles/edxapp/defaults/main.yml | 8 ++++++++ playbooks/roles/edxapp/tasks/deploy.yml | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed23a7921f3..f2d12b66030 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +- Role: edxapp + - Added creation of enterprise_worker user to provisioning. This user is used by the edx-enterprise package when making API requests to Open edX IDAs. + - Role: forum - Added `FORUM_REBUILD_INDEX` to rebuild the ElasticSearch index from the database, when enabled. Default: `False`. diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 4d066923987..1d3076930e4 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -220,3 +220,6 @@ COMMON_OIDC_ISSUER: '{{ COMMON_OAUTH_URL_ROOT }}' COMMON_JWT_AUDIENCE: 'SET-ME-PLEASE' COMMON_JWT_ISSUER: '{{ COMMON_OIDC_ISSUER }}' COMMON_JWT_SECRET_KEY: 'SET-ME-PLEASE' + +# Set worker user default +CREATE_SERVICE_WORKER_USERS: True diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 64a8e2f56f7..ac3b9d776a3 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -692,6 +692,7 @@ EDXAPP_BLOCK_STRUCTURES_SETTINGS: # Configuration settings needed for the LMS to communicate with the Enterprise service. EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_ROOT_URL }}/enterprise/api/v1" +EDXAPP_ENTERPRISE_SERVICE_WORKER_EMAIL: "enterprise_worker@example.com" EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME: "enterprise_worker" EDXAPP_ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES: @@ -1300,3 +1301,10 @@ edxapp_cms_variant: cms # Worker Settings worker_django_settings_module: '{{ EDXAPP_SETTINGS }}' + +# Add default service worker users +SERVICE_WORKER_USERS: + - email: "{{ EDXAPP_ENTERPRISE_SERVICE_WORKER_EMAIL }}" + username: "{{ EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME }}" + is_staff: true + is_superuser: false diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index b7f72d163e8..5eb2b341748 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -417,3 +417,22 @@ become_user: "{{ common_web_user }}" tags: - manage + +- name: build service worker users list + set_fact: + service_worker_users: "{{ [SERVICE_WORKER_USERS] }}" + when: CREATE_SERVICE_WORKER_USERS + tags: + - manage + - manage:db + +- name: create service worker users + shell: "{{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} --service-variant lms manage_user {{ item.username}} {{ item.email }} --unusable-password {% if item.is_staff %} --staff{% endif %}" + args: + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ common_web_user }}" + with_items: "{{ service_worker_users }}" + when: CREATE_SERVICE_WORKER_USERS + tags: + - manage + - manage:db From 05cca2b077cb8f9ab2837be2532924b4eaad985d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 9 Aug 2017 06:04:31 -0400 Subject: [PATCH 10/60] sandbox.sh will use command-line args for the big playbook run (cherry picked from commit e198263be149e805605cc631b9595f546e00e1fe) --- util/install/sandbox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install/sandbox.sh b/util/install/sandbox.sh index 04915db9c5d..21c19b99d50 100644 --- a/util/install/sandbox.sh +++ b/util/install/sandbox.sh @@ -89,4 +89,4 @@ sudo -H pip install -r requirements.txt ## ## Run the edx_sandbox.yml playbook in the configuration/playbooks directory ## -cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS +cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS "$@" From 8f6dd7bfcc598b83d0fc3b8cfa1c27638b0e43fc Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 14 Aug 2017 13:14:57 -0400 Subject: [PATCH 11/60] Vagrantfile updates for ginkgo.1 --- vagrant/release/devstack/Vagrantfile | 3 ++- vagrant/release/fullstack/Vagrantfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index 49bfc8c7691..8e1c6f69c90 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -45,6 +45,7 @@ end openedx_releases = { "open-release/ginkgo.master" => "ginkgo-devstack-2017-07-14", "open-release/ginkgo.1rc1" => "ginkgo-devstack-2017-07-14", + "open-release/ginkgo.1" => "ginkgo-devstack-2017-07-14", "open-release/ficus.master" => "ficus-devstack-2017-02-07", "open-release/ficus.1rc1" => "ficus-devstack-2017-01-11", @@ -69,7 +70,7 @@ openedx_releases = { # Cypress is deprecated and unsupported # Birch is deprecated and unsupported } -openedx_releases.default = "ficus-devstack-2017-02-07" +openedx_releases.default = "ginkgo-devstack-2017-07-14" openedx_release = ENV['OPENEDX_RELEASE'] boxname = ENV['OPENEDX_BOXNAME'] diff --git a/vagrant/release/fullstack/Vagrantfile b/vagrant/release/fullstack/Vagrantfile index a9241c3c8ee..5bb7c51a83b 100644 --- a/vagrant/release/fullstack/Vagrantfile +++ b/vagrant/release/fullstack/Vagrantfile @@ -10,6 +10,7 @@ CPU_COUNT = 2 # a Vagrant box from the internet. openedx_releases = { "open-release/ginkgo.1rc1" => "ginkgo-fullstack-2017-07-14", + "open-release/ginkgo.1" => "ginkgo-fullstack-2017-08-14", "open-release/ficus.1rc3" => "ficus-fullstack-2017-02-07", "open-release/ficus.1rc4" => "ficus-fullstack-2017-02-15", @@ -40,7 +41,7 @@ openedx_releases = { # Cypress is deprecated and unsupported # Birch is deprecated and unsupported } -openedx_releases.default = "ficus-fullstack-2017-04-20" +openedx_releases.default = "ginkgo-fullstack-2017-08-14" openedx_release = ENV['OPENEDX_RELEASE'] boxname = ENV['OPENEDX_BOXNAME'] From 67b4c16b3147c5f73845afc411e25ea18c811631 Mon Sep 17 00:00:00 2001 From: Brian Beggs Date: Tue, 11 Jul 2017 14:48:10 -0400 Subject: [PATCH 12/60] Adding all defaults required for VIDEO_IMAGE_SETTINGS (cherry picked from commit f92a3613a66c22201125367e32e1366b2e378b59) --- playbooks/roles/edxapp/defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index ac3b9d776a3..8b5c9665e61 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -673,6 +673,10 @@ EDXAPP_VIDEO_IMAGE_MAX_AGE: 31536000 EDXAPP_VIDEO_IMAGE_SETTINGS: VIDEO_IMAGE_MAX_BYTES : 2097152 VIDEO_IMAGE_MIN_BYTES : 2048 + STORAGE_KWARGS: + location: "{{ edxapp_media_dir }}/" + base_url: "{{ EDXAPP_MEDIA_URL }}/" + DIRECTORY_PREFIX: 'video-images/' # Course Block Structures EDXAPP_BLOCK_STRUCTURES_SETTINGS: From bad6a43debf784deff5aba2a13a3f09840e1660e Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Thu, 17 Aug 2017 12:47:01 -0400 Subject: [PATCH 13/60] Updated forum tags to update env settings on configuration Added the `install:configuration` tag to the step that writes the forum_env to disk so that updated mongodb credentials are properly propagated to the service during deployments. --- playbooks/roles/forum/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/forum/tasks/main.yml b/playbooks/roles/forum/tasks/main.yml index 07cb98e7362..0ce79491eb1 100644 --- a/playbooks/roles/forum/tasks/main.yml +++ b/playbooks/roles/forum/tasks/main.yml @@ -56,6 +56,7 @@ tags: - install - install:base + - install:configuration - name: create {{ forum_data_dir }} file: From 3d664028e1b0b92458c023a5c8308bc74d6a9009 Mon Sep 17 00:00:00 2001 From: sar Date: Mon, 31 Jul 2017 15:08:42 -0400 Subject: [PATCH 14/60] Added a way to override having the aws role be installed. In cases where a local host is being used, aws role is not needed. Default remains true though. --- playbooks/edx-east/edxapp.yml | 3 ++- playbooks/roles/common_vars/defaults/main.yml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/edx-east/edxapp.yml b/playbooks/edx-east/edxapp.yml index 70b8ce2c09d..891625b4256 100644 --- a/playbooks/edx-east/edxapp.yml +++ b/playbooks/edx-east/edxapp.yml @@ -7,7 +7,8 @@ CLUSTER_NAME: 'edxapp' serial: "{{ serial_count }}" roles: - - aws + - role: aws + when: COMMON_ENABLE_AWS_ROLE - role: automated AUTOMATED_USERS: "{{ EDXAPP_AUTOMATED_USERS | default({}) }}" - role: nginx diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 1d3076930e4..2b36bedec7b 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -223,3 +223,5 @@ COMMON_JWT_SECRET_KEY: 'SET-ME-PLEASE' # Set worker user default CREATE_SERVICE_WORKER_USERS: True + +COMMON_ENABLE_AWS_ROLE: true From 4bb7064668701dc7e54174512412f96dd1c5386a Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Mon, 25 Sep 2017 11:42:09 -0400 Subject: [PATCH 15/60] Exposed configuration of atomic requests Added the option of overriding atomic requests without having to override the entire database configuration structure. --- playbooks/roles/edxapp/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 8b5c9665e61..93c468a8e9b 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -79,6 +79,7 @@ EDXAPP_MYSQL_PASSWORD: 'password' EDXAPP_MYSQL_PASSWORD_READ_ONLY: 'password' EDXAPP_MYSQL_PASSWORD_ADMIN: 'password' EDXAPP_MYSQL_OPTIONS: {} +EDXAPP_MYSQL_ATOMIC_REQUESTS: True EDXAPP_MYSQL_REPLICA_DB_NAME: "{{ EDXAPP_MYSQL_DB_NAME }}" EDXAPP_MYSQL_REPLICA_USER: "{{ EDXAPP_MYSQL_USER }}" EDXAPP_MYSQL_REPLICA_PASSWORD: "{{ EDXAPP_MYSQL_PASSWORD }}" @@ -825,7 +826,7 @@ edxapp_databases: PASSWORD: "{{ EDXAPP_MYSQL_PASSWORD }}" HOST: "{{ EDXAPP_MYSQL_HOST }}" PORT: "{{ EDXAPP_MYSQL_PORT }}" - ATOMIC_REQUESTS: True + ATOMIC_REQUESTS: "{{ EDXAPP_MYSQL_ATOMIC_REQUESTS }}" CONN_MAX_AGE: "{{ EDXAPP_MYSQL_CONN_MAX_AGE }}" OPTIONS: "{{ EDXAPP_MYSQL_OPTIONS }}" student_module_history: From af71be87a2735c4618fa253ed5c96f51e63bc74b Mon Sep 17 00:00:00 2001 From: Zia Fazal Date: Tue, 19 Sep 2017 18:52:31 +0500 Subject: [PATCH 16/60] ignore node_modules directory while compiling python files signle quoted regex --- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 5eb2b341748..667d261d260 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -300,7 +300,7 @@ - install:app-requirements - name: compiling all py files in the edx-platform repo - shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x .git/.* {{ edxapp_code_dir }}" + shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}" become_user: "{{ edxapp_user }}" tags: - install From 9d6eb34f74f72e2da4da2e045215a6ccb311a745 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Dec 2017 08:26:51 -0500 Subject: [PATCH 17/60] Vagrantfile changes for ginkgo.2 --- vagrant/release/devstack/Vagrantfile | 1 + vagrant/release/fullstack/Vagrantfile | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index 8e1c6f69c90..f24a2631b31 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -46,6 +46,7 @@ openedx_releases = { "open-release/ginkgo.master" => "ginkgo-devstack-2017-07-14", "open-release/ginkgo.1rc1" => "ginkgo-devstack-2017-07-14", "open-release/ginkgo.1" => "ginkgo-devstack-2017-07-14", + "open-release/ginkgo.2" => "ginkgo-devstack-2017-07-14", "open-release/ficus.master" => "ficus-devstack-2017-02-07", "open-release/ficus.1rc1" => "ficus-devstack-2017-01-11", diff --git a/vagrant/release/fullstack/Vagrantfile b/vagrant/release/fullstack/Vagrantfile index 5bb7c51a83b..3d20acf56eb 100644 --- a/vagrant/release/fullstack/Vagrantfile +++ b/vagrant/release/fullstack/Vagrantfile @@ -11,6 +11,7 @@ CPU_COUNT = 2 openedx_releases = { "open-release/ginkgo.1rc1" => "ginkgo-fullstack-2017-07-14", "open-release/ginkgo.1" => "ginkgo-fullstack-2017-08-14", + "open-release/ginkgo.2" => "ginkgo-fullstack-2017-12-14", "open-release/ficus.1rc3" => "ficus-fullstack-2017-02-07", "open-release/ficus.1rc4" => "ficus-fullstack-2017-02-15", @@ -41,7 +42,7 @@ openedx_releases = { # Cypress is deprecated and unsupported # Birch is deprecated and unsupported } -openedx_releases.default = "ginkgo-fullstack-2017-08-14" +openedx_releases.default = "ginkgo-fullstack-2017-12-14" openedx_release = ENV['OPENEDX_RELEASE'] boxname = ENV['OPENEDX_BOXNAME'] From e1dfcae17f54791cb45cc1945ae632a84ee793f2 Mon Sep 17 00:00:00 2001 From: Myles Date: Fri, 12 Jan 2018 10:26:03 +0800 Subject: [PATCH 18/60] Override DISCOVERY_VERSION in install script Override branch name `master` with the correct release branch when installing (cherry picked from commit 5dc8fdb6d7cb29d0ec2937e236298e534c8db7f0) --- util/install/sandbox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/install/sandbox.sh b/util/install/sandbox.sh index 21c19b99d50..380e5fb2a78 100644 --- a/util/install/sandbox.sh +++ b/util/install/sandbox.sh @@ -52,6 +52,7 @@ VERSION_VARS=( ANALYTICS_API_VERSION ECOMMERCE_VERSION ECOMMERCE_WORKER_VERSION + DISCOVERY_VERSION ) for var in ${VERSION_VARS[@]}; do From 78406b8a482de49c1bc7800d6b87e3dab28764d5 Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Wed, 28 Mar 2018 13:59:10 -0400 Subject: [PATCH 19/60] Upgrade setuptools, pip, and virtualenv (cherry picked from commit 2cc32cffe7d106375560aae715fb19d4d50343f7) --- docker/build/go-agent/Dockerfile | 8 ++++---- playbooks/roles/common_vars/defaults/main.yml | 8 ++++---- util/install/ansible-bootstrap.sh | 6 +++--- util/install/sandbox.sh | 6 +++--- util/vpc-tools/abbey.py | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docker/build/go-agent/Dockerfile b/docker/build/go-agent/Dockerfile index ee24d29bd7c..57bad050d05 100644 --- a/docker/build/go-agent/Dockerfile +++ b/docker/build/go-agent/Dockerfile @@ -58,13 +58,13 @@ RUN printf "\ngo ALL=(ALL:ALL) NOPASSWD: /usr/bin/pip3, /usr/local/bin/pip3 # Upgrade pip and setup tools. Needed for Ansible 2.x RUN \ - pip install --upgrade pip==8.1.2 && \ - pip3 install --upgrade pip==8.1.2 && \ + pip install --upgrade pip==9.0.3 && \ + pip3 install --upgrade pip==9.0.3 && \ #pip may have moved from /usr/bin/ to /usr/local/bin/. This clears bash's path cache. hash -r && \ # upgrade setuptools early to avoid no distribution errors - pip install --upgrade setuptools==24.0.3 && \ - pip3 install --upgrade setuptools==24.0.3 + pip install --upgrade setuptools==39.0.1 && \ + pip3 install --upgrade setuptools==39.0.1 # Install AWS command-line interface - for AWS operations in a go-agent task. diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 2b36bedec7b..100895be840 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -125,10 +125,10 @@ old_python_debian_pkgs: - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1" common_pip_pkgs: - - pip==8.1.2 - - setuptools==24.0.3 - - virtualenv==15.0.2 - - virtualenvwrapper==4.7.1 + - pip==9.0.3 + - setuptools==39.0.1 + - virtualenv==15.2.0 + - virtualenvwrapper==4.8.2 common_web_user: www-data common_web_group: www-data diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 5e1c120cc86..e608bffaa18 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -40,9 +40,9 @@ fi # # Bootstrapping constants # -VIRTUAL_ENV_VERSION="15.0.2" -PIP_VERSION="8.1.2" -SETUPTOOLS_VERSION="24.0.3" +VIRTUAL_ENV_VERSION="15.2.0" +PIP_VERSION="9.0.3" +SETUPTOOLS_VERSION="39.0.1" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" ANSIBLE_DIR="/tmp/ansible" diff --git a/util/install/sandbox.sh b/util/install/sandbox.sh index 380e5fb2a78..da3d42536d6 100644 --- a/util/install/sandbox.sh +++ b/util/install/sandbox.sh @@ -32,9 +32,9 @@ sudo apt-get upgrade -y ## Install system pre-requisites ## sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ -sudo pip install --upgrade pip==8.1.2 -sudo pip install --upgrade setuptools==24.0.3 -sudo -H pip install --upgrade virtualenv==15.0.2 +sudo pip install --upgrade pip==9.0.3 +sudo pip install --upgrade setuptools==39.0.1 +sudo -H pip install --upgrade virtualenv==15.2.0 ## ## Overridable version variables in the playbooks. Each can be overridden diff --git a/util/vpc-tools/abbey.py b/util/vpc-tools/abbey.py index 04e22b38bd1..11f61b5a920 100644 --- a/util/vpc-tools/abbey.py +++ b/util/vpc-tools/abbey.py @@ -327,9 +327,9 @@ def create_instance_args(): # # Bootstrapping constants # -VIRTUAL_ENV_VERSION="15.0.2" -PIP_VERSION="8.1.2" -SETUPTOOLS_VERSION="24.0.3" +VIRTUAL_ENV_VERSION="15.2.0" +PIP_VERSION="9.0.3" +SETUPTOOLS_VERSION="39.0.1" EDX_PPA="deb http://ppa.edx.org precise main" EDX_PPA_KEY_SERVER="keyserver.ubuntu.com" EDX_PPA_KEY_ID="B41E5E3969464050" From d6a17a27893d345367187eecff6fdb13e3350774 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 11 Jul 2018 09:20:15 -0400 Subject: [PATCH 20/60] Add git status to this diagnostic helper (cherry picked from commit 834adf079bcea9c65cfb0641d567e80c2f1f020c) --- playbooks/roles/edx_ansible/templates/show-repo-heads.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/edx_ansible/templates/show-repo-heads.j2 b/playbooks/roles/edx_ansible/templates/show-repo-heads.j2 index 3ddfa4e9b83..28a3d18d0c4 100644 --- a/playbooks/roles/edx_ansible/templates/show-repo-heads.j2 +++ b/playbooks/roles/edx_ansible/templates/show-repo-heads.j2 @@ -2,10 +2,13 @@ # # Display the head commit for all our git repos +echo "With status..." + # Look in .git directories. Only two levels down to avoid virtualenvs. for d in {{ COMMON_APP_DIR }}/*/*/.git; do d=$(dirname $d) echo "---- $d ----" git -C $d log -1 --format='%ci (%h) %cn: %s %d' + git -C $d status --short done From 90a27efc50490038581eca49abd6956f0a8066da Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 12 Jul 2018 11:50:41 -0400 Subject: [PATCH 21/60] sandbox.sh -> native.sh, with improvements These are the changes from pull requests #4682 and #4683, applied to ginkgo. --- util/install/native.sh | 131 ++++++++++++++++++++++++++++++++++++++++ util/install/sandbox.sh | 96 +++-------------------------- 2 files changed, 138 insertions(+), 89 deletions(-) create mode 100644 util/install/native.sh diff --git a/util/install/native.sh b/util/install/native.sh new file mode 100644 index 00000000000..220b9b1277a --- /dev/null +++ b/util/install/native.sh @@ -0,0 +1,131 @@ +#!/bin/bash +## +## Installs the pre-requisites for running Open edX on a single Ubuntu 16.04 +## instance. This script is provided as a convenience and any of these +## steps could be executed manually. +## +## Note that this script requires that you have the ability to run +## commands as root via sudo. Caveat Emptor! +## + +## +## Sanity checks +## + +if [[ ! $OPENEDX_RELEASE ]]; then + echo "You must define OPENEDX_RELEASE" + exit +fi + +if [[ `lsb_release -rs` != "16.04" ]]; then + echo "This script is only known to work on Ubuntu 16.04, exiting..." + exit +fi + +## +## Log what's happening +## + +mkdir -p logs +log_file=logs/install-$(date +%Y%m%d-%H%M%S).log +exec > >(tee $log_file) 2>&1 +echo "Capturing output to $log_file" +echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')" + +function finish { + echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')" +} +trap finish EXIT + +echo "Installing release '$OPENEDX_RELEASE'" + +## +## Set ppa repository source for gcc/g++ 4.8 in order to install insights properly +## +sudo apt-get install -y python-software-properties +sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + +## +## Update and Upgrade apt packages +## +sudo apt-get update -y +sudo apt-get upgrade -y + +## +## Install system pre-requisites +## +sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ +sudo pip install --upgrade pip==9.0.3 +sudo pip install --upgrade setuptools==39.0.1 +sudo -H pip install --upgrade virtualenv==15.2.0 + +## +## Overridable version variables in the playbooks. Each can be overridden +## individually, or with $OPENEDX_RELEASE. +## +VERSION_VARS=( + edx_platform_version + certs_version + forum_version + xqueue_version + configuration_version + demo_version + NOTIFIER_VERSION + INSIGHTS_VERSION + ANALYTICS_API_VERSION + ECOMMERCE_VERSION + ECOMMERCE_WORKER_VERSION + DISCOVERY_VERSION +) + +for var in ${VERSION_VARS[@]}; do + # Each variable can be overridden by a similarly-named environment variable, + # or OPENEDX_RELEASE, if provided. + ENV_VAR=$(echo $var | tr '[:lower:]' '[:upper:]') + eval override=\${$ENV_VAR-\$OPENEDX_RELEASE} + if [ -n "$override" ]; then + EXTRA_VARS="-e $var=$override $EXTRA_VARS" + fi +done + +# my-passwords.yml is the file made by generate-passwords.sh. +if [[ -f my-passwords.yml ]]; then + EXTRA_VARS="-e@$(pwd)/my-passwords.yml $EXTRA_VARS" +fi + +CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} + +## +## Clone the configuration repository and run Ansible +## +cd /var/tmp +git clone https://github.com/edx/configuration +cd configuration +git checkout $CONFIGURATION_VERSION +git pull + +## +## Install the ansible requirements +## +cd /var/tmp/configuration +sudo -H pip install -r requirements.txt + +## +## Run the edx_sandbox.yml playbook in the configuration/playbooks directory +## +cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS "$@" +ansible_status=$? + +if [[ $ansible_status -ne 0 ]]; then + echo " " + echo "========================================" + echo "Ansible failed!" + echo "----------------------------------------" + echo "If you need help, see https://open.edx.org/getting-help ." + echo "When asking for help, please provide as much information as you can." + echo "These might be helpful:" + echo " Your log file is at $log_file" + echo " Your environment:" + env | egrep -i 'version|release' | sed -e 's/^/ /' + echo "========================================" +fi diff --git a/util/install/sandbox.sh b/util/install/sandbox.sh index da3d42536d6..fe81ec15932 100644 --- a/util/install/sandbox.sh +++ b/util/install/sandbox.sh @@ -1,93 +1,11 @@ #!/bin/bash ## -## Installs the pre-requisites for running edX on a single Ubuntu 12.04 -## instance. This script is provided as a convenience and any of these -## steps could be executed manually. +## This was the old name for native.sh. If someone runs it, tell them +## where the file went. ## -## Note that this script requires that you have the ability to run -## commands as root via sudo. Caveat Emptor! -## - -## -## Sanity check -## -if [[ `lsb_release -rs` != "16.04" ]]; then - echo "This script is only known to work on Ubuntu 16.04, exiting..."; - exit; -fi - -## -## Set ppa repository source for gcc/g++ 4.8 in order to install insights properly -## -sudo apt-get install -y python-software-properties -sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -## -## Update and Upgrade apt packages -## -sudo apt-get update -y -sudo apt-get upgrade -y - -## -## Install system pre-requisites -## -sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ -sudo pip install --upgrade pip==9.0.3 -sudo pip install --upgrade setuptools==39.0.1 -sudo -H pip install --upgrade virtualenv==15.2.0 - -## -## Overridable version variables in the playbooks. Each can be overridden -## individually, or with $OPENEDX_RELEASE. -## -VERSION_VARS=( - edx_platform_version - certs_version - forum_version - xqueue_version - configuration_version - demo_version - NOTIFIER_VERSION - INSIGHTS_VERSION - ANALYTICS_API_VERSION - ECOMMERCE_VERSION - ECOMMERCE_WORKER_VERSION - DISCOVERY_VERSION -) - -for var in ${VERSION_VARS[@]}; do - # Each variable can be overridden by a similarly-named environment variable, - # or OPENEDX_RELEASE, if provided. - ENV_VAR=$(echo $var | tr '[:lower:]' '[:upper:]') - eval override=\${$ENV_VAR-\$OPENEDX_RELEASE} - if [ -n "$override" ]; then - EXTRA_VARS="-e $var=$override $EXTRA_VARS" - fi -done - -# my-passwords.yml is the file made by generate-passwords.sh. -if [[ -f my-passwords.yml ]]; then - EXTRA_VARS="-e@$(pwd)/my-passwords.yml $EXTRA_VARS" -fi - -CONFIGURATION_VERSION=${CONFIGURATION_VERSION-${OPENEDX_RELEASE-master}} - -## -## Clone the configuration repository and run Ansible -## -cd /var/tmp -git clone https://github.com/edx/configuration -cd configuration -git checkout $CONFIGURATION_VERSION -git pull - -## -## Install the ansible requirements -## -cd /var/tmp/configuration -sudo -H pip install -r requirements.txt - -## -## Run the edx_sandbox.yml playbook in the configuration/playbooks directory -## -cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS "$@" +echo +echo "The sandbox.sh script has been renamed to native.sh" +echo "Let the author of your instructions know, or send us an email at oscm@edx.org" +echo "Re-run your command with native.sh instead of sandbox.sh" +echo From 8a0b40ad12114d76d14ca92d07587220b9d6b3ca Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 10 Jul 2018 15:50:49 -0400 Subject: [PATCH 22/60] Check that git working trees are clean (cherry picked from commit ec3079910a0f31c67f06c79a1e80c5bbdbe414a7) --- playbooks/roles/git_clone/tasks/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/playbooks/roles/git_clone/tasks/main.yml b/playbooks/roles/git_clone/tasks/main.yml index d79d9bd8efb..79efe963a35 100644 --- a/playbooks/roles/git_clone/tasks/main.yml +++ b/playbooks/roles/git_clone/tasks/main.yml @@ -54,6 +54,20 @@ - install - install:code +- name: Check that working tree is clean + shell: test ! -e "{{ item }}" || git -C "{{ item }}" status --porcelain --untracked-files=no + register: dirty_files + # Using the map here means that the items will only be the DESTINATION strings, + # rather than the full GIT_REPOS structures, which have data we don't want to log, + # so we don't have to use no_log on this task. + with_items: "{{ GIT_REPOS | map(attribute='DESTINATION') | list }}" + +- name: Require a clean working tree + fail: + msg: "There are modified files in {{ item.item }}: {{ item.stdout }}" + when: item.stdout|length > 0 + with_items: "{{ dirty_files.results }}" + - name: Checkout code over ssh git: repo: "git@{{ item.DOMAIN }}:{{ item.PATH }}/{{ item.REPO }}" From 4964ff77c327cff10fc2cea19c2df9b863d44e0b Mon Sep 17 00:00:00 2001 From: terapyon Date: Mon, 27 Aug 2018 17:25:52 +0900 Subject: [PATCH 23/60] added README for my config --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 48618d20dbb..ea7fcb87cf3 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,12 @@ Configuration Management ######################## +About +********** + +fro CMScom config + + Introduction ************ From f6355d7905018c41e62452ddfde2709535e8c48b Mon Sep 17 00:00:00 2001 From: terapyon Date: Mon, 27 Aug 2018 17:33:40 +0900 Subject: [PATCH 24/60] added edx remote config for CMScom --- playbooks/edx_cmscom.yml | 76 ++++++++++++++++++++ playbooks/roles/edxremote/defaults/main.yml | 63 +++++++++++++++++ playbooks/roles/edxremote/meta/main.yml | 4 ++ playbooks/roles/edxremote/tasks/main.yml | 77 +++++++++++++++++++++ 4 files changed, 220 insertions(+) create mode 100644 playbooks/edx_cmscom.yml create mode 100644 playbooks/roles/edxremote/defaults/main.yml create mode 100644 playbooks/roles/edxremote/meta/main.yml create mode 100644 playbooks/roles/edxremote/tasks/main.yml diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml new file mode 100644 index 00000000000..5da2e885f94 --- /dev/null +++ b/playbooks/edx_cmscom.yml @@ -0,0 +1,76 @@ +--- + +# Example sandbox configuration +# for single server community +# installs + +- name: Bootstrap instance(s) + hosts: all + gather_facts: no + become: True + roles: + - python + +- name: Configure instance(s) + hosts: all + become: True + gather_facts: True + vars: + migrate_db: "yes" + EDXAPP_LMS_NGINX_PORT: '80' + edx_platform_version: 'master' + # Set to false if deployed behind another proxy/load balancer. + NGINX_SET_X_FORWARDED_HEADERS: True + DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}' + roles: + - role: swapfile + SWAPFILE_SIZE: 4GB + - role: nginx + nginx_sites: + - certs + - cms + - lms + - forum + - xqueue + - ecommerce + nginx_default_sites: + - lms +# - role: edxlocal +# when: EDXAPP_MYSQL_HOST == 'localhost' + - role: edxremote + - role: memcache + when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" + - role: mongo + when: "'localhost' in EDXAPP_MONGO_HOSTS" + - role: rabbitmq + rabbitmq_ip: 127.0.0.1 + - role: edxapp + celery_worker: True + - edxapp + - ecommerce + - role: ecomworker + ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 + - analytics_api + - insights + # not ready yet: - edx_notes_api + - demo + - oauth_client_setup + - oraclejdk + - role: elasticsearch + when: "'localhost' in EDXAPP_ELASTIC_SEARCH_CONFIG|map(attribute='host')" + - forum + - discovery + - role: notifier + NOTIFIER_DIGEST_TASK_INTERVAL: 5 + - role: xqueue + update_users: True + - certs + - edx_ansible + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic + when: COMMON_ENABLE_NEWRELIC + - role: postfix_queue + when: POSTFIX_QUEUE_EXTERNAL_SMTP_HOST != '' diff --git a/playbooks/roles/edxremote/defaults/main.yml b/playbooks/roles/edxremote/defaults/main.yml new file mode 100644 index 00000000000..5f368163398 --- /dev/null +++ b/playbooks/roles/edxremote/defaults/main.yml @@ -0,0 +1,63 @@ +--- +edxlocal_debian_pkgs: + - postfix + - libjpeg-dev + +edxlocal_databases: + - "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}" + - "{{ INSIGHTS_DATABASE_NAME | default(None) }}" + - "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}" + - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}" + - "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}" + - "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}" + - "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}" + - "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}" + - "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}" + - "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}" + +edxlocal_database_users: + - { + db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}", + user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}", + pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}" + } + - { + db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}", + user: "{{ INSIGHTS_DATABASE_USER | default(None) }}", + pass: "{{ INSIGHTS_DATABASE_PASSWORD | default(None) }}" + } + - { + db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}", + user: "{{ XQUEUE_MYSQL_USER | default(None) }}", + pass: "{{ XQUEUE_MYSQL_PASSWORD | default(None) }}" + } + - { + db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}", + user: "{{ EDXAPP_MYSQL_USER | default(None) }}", + pass: "{{ EDXAPP_MYSQL_PASSWORD | default(None) }}" + } + - { + db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}", + user: "{{ EDXAPP_MYSQL_CSMH_USER | default(None) }}", + pass: "{{ EDXAPP_MYSQL_CSMH_PASSWORD | default(None) }}" + } + - { + db: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_NAME | default(None) }}", + user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_USER | default(None) }}", + pass: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_PASSWORD | default(None) }}" + } + - { + db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}", + user: "{{ HIVE_METASTORE_DATABASE_USER | default(None) }}", + pass: "{{ HIVE_METASTORE_DATABASE_PASSWORD | default(None) }}" + } + - { + db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}", + user: "{{ CREDENTIALS_MYSQL_USER | default(None) }}", + pass: "{{ CREDENTIALS_MYSQL_PASSWORD | default(None) }}" + } + - { + db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}", + user: "{{ DISCOVERY_MYSQL_USER | default(None) }}", + pass: "{{ DISCOVERY_MYSQL_PASSWORD | default(None) }}" + } diff --git a/playbooks/roles/edxremote/meta/main.yml b/playbooks/roles/edxremote/meta/main.yml new file mode 100644 index 00000000000..edc87cdd654 --- /dev/null +++ b/playbooks/roles/edxremote/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - common +# - mysql diff --git a/playbooks/roles/edxremote/tasks/main.yml b/playbooks/roles/edxremote/tasks/main.yml new file mode 100644 index 00000000000..96340f7392c --- /dev/null +++ b/playbooks/roles/edxremote/tasks/main.yml @@ -0,0 +1,77 @@ +# Installs packages to run edx locally on a single instance +# +--- +- name: Install packages needed for single server + apt: + name: "{{ item }}" + install_recommends: yes + state: present + with_items: "{{ edxlocal_debian_pkgs }}" + +# TODO: Add a test to make sure mysql is running. + +- name: create databases + mysql_db: + db: "{{ item }}" + state: present + encoding: utf8 + when: item != None and item != '' + with_items: "{{ edxlocal_databases }}" + +- name: create database users + mysql_user: + name: "{{ item.user }}" + password: "{{ item.pass }}" + priv: "{{ item.db }}.*:ALL" + append_privs: yes + when: item.db != None and item.db != '' + with_items: "{{ edxlocal_database_users }}" + +- name: setup the migration db user + mysql_user: + name: "{{ COMMON_MYSQL_MIGRATE_USER }}" + password: "{{ COMMON_MYSQL_MIGRATE_PASS }}" + priv: "{{ item }}.*:ALL" + append_privs: yes + when: item != None and item != '' + with_items: "{{ edxlocal_databases }}" + +- name: create api user for the analytics api + mysql_user: + name: "api001" + password: "{{ ANALYTICS_API_DATABASES.default.PASSWORD }}" + priv: '{{ ANALYTICS_API_DATABASES.default.NAME }}.*:ALL/reports.*:SELECT' + when: ANALYTICS_API_SERVICE_CONFIG is defined + +- name: create read-only reports user for the analytics-api + mysql_user: + name: reports001 + password: "{{ ANALYTICS_API_DATABASES.reports.PASSWORD }}" + priv: '{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:SELECT' + when: ANALYTICS_API_SERVICE_CONFIG is defined + +- name: create a database for the hive metastore + mysql_db: + db: "{{ HIVE_METASTORE_DATABASE.name }}" + state: "present" + encoding: "latin1" + when: HIVE_METASTORE_DATABASE is defined + +- name: setup the edx-notes-api db user + mysql_user: + name: "{{ EDX_NOTES_API_MYSQL_DB_USER }}" + password: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}" + priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE" + when: EDX_NOTES_API_MYSQL_DB_USER is defined + +- name: setup the read-only db user + mysql_user: + name: "{{ COMMON_MYSQL_READ_ONLY_USER }}" + password: "{{ COMMON_MYSQL_READ_ONLY_PASS }}" + priv: "*.*:ALL" + +- name: setup the admin db user + mysql_user: + name: "{{ COMMON_MYSQL_ADMIN_USER }}" + password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + priv: "*.*:CREATE USER" From dc6f0b91466d8c0d10ba123a6aec4a81b664bf4a Mon Sep 17 00:00:00 2001 From: terapyon Date: Mon, 27 Aug 2018 17:36:52 +0900 Subject: [PATCH 25/60] remove ec and insite --- playbooks/edx_cmscom.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 5da2e885f94..69e69689ac9 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -32,7 +32,7 @@ - lms - forum - xqueue - - ecommerce +# - ecommerce nginx_default_sites: - lms # - role: edxlocal @@ -47,11 +47,11 @@ - role: edxapp celery_worker: True - edxapp - - ecommerce - - role: ecomworker - ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 +# - ecommerce +# - role: ecomworker +# ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 - analytics_api - - insights +# - insights # not ready yet: - edx_notes_api - demo - oauth_client_setup From bd6b1f2ed95550b3067ad7eb2ae725d4ecdf5910 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 28 Aug 2018 12:13:51 +0900 Subject: [PATCH 26/60] Update generate-passwords.sh bugfix: renamed repository name. --- util/install/generate-passwords.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/install/generate-passwords.sh b/util/install/generate-passwords.sh index 417a7e9c39e..0b3055220df 100755 --- a/util/install/generate-passwords.sh +++ b/util/install/generate-passwords.sh @@ -3,8 +3,9 @@ # Read a list of Ansible variables that should have generated values, and make # a new file just like it, with the generated values. -TARGET=${OPENEDX_RELEASE-master} -wget -q "https://raw.githubusercontent.com/edx/configuration/$TARGET/playbooks/sample_vars/passwords.yml" -O passwords-template.yml +#TARGET=${OPENEDX_RELEASE-master} +TARGET=${OPENEDX_RELEASE} +wget -q "https://raw.githubusercontent.com/cmscom/configuration/$TARGET/playbooks/sample_vars/passwords.yml" -O passwords-template.yml while IFS= read -r line; do # Make a random string. SECRET_KEY's should be longer. From 60e3c0899402616c157db7f8b56554f07fe299c7 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 28 Aug 2018 12:36:22 +0900 Subject: [PATCH 27/60] Update native.sh bugfix: change clone address to cmscom --- util/install/native.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install/native.sh b/util/install/native.sh index 220b9b1277a..144fe639362 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -99,7 +99,7 @@ CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} ## Clone the configuration repository and run Ansible ## cd /var/tmp -git clone https://github.com/edx/configuration +git clone https://github.com/cmscom/configuration cd configuration git checkout $CONFIGURATION_VERSION git pull From f932debf3f5d5cdc17f5d9a686ddb338adfb2bdb Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 28 Aug 2018 15:57:10 +0900 Subject: [PATCH 28/60] Update native.sh --- util/install/native.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/install/native.sh b/util/install/native.sh index 144fe639362..8d811585d62 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -51,6 +51,9 @@ sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update -y sudo apt-get upgrade -y +## added require modules. @cmscom +sudo apt-get install -y mariadb-client + ## ## Install system pre-requisites ## From 0f69d104f4fa5a109b7ca6814aae1118c289a65e Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 28 Aug 2018 16:56:59 +0900 Subject: [PATCH 29/60] Update native.sh bugfix: switch yml --- util/install/native.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install/native.sh b/util/install/native.sh index 8d811585d62..97a466fc0ef 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -116,7 +116,7 @@ sudo -H pip install -r requirements.txt ## ## Run the edx_sandbox.yml playbook in the configuration/playbooks directory ## -cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS "$@" +cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_cmscom.yml -i "localhost," $EXTRA_VARS "$@" ansible_status=$? if [[ $ansible_status -ne 0 ]]; then From 5d49eb4c1ad72845f3c12eb7392d08cebc23faf0 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 28 Aug 2018 17:14:57 +0900 Subject: [PATCH 30/60] Update main.yml bugfix: added support the remote database. --- playbooks/roles/edxremote/tasks/main.yml | 34 +++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxremote/tasks/main.yml b/playbooks/roles/edxremote/tasks/main.yml index 96340f7392c..612de9ac08b 100644 --- a/playbooks/roles/edxremote/tasks/main.yml +++ b/playbooks/roles/edxremote/tasks/main.yml @@ -1,4 +1,4 @@ -# Installs packages to run edx locally on a single instance + # Installs packages to run edx locally on a single instance # --- - name: Install packages needed for single server @@ -12,6 +12,9 @@ - name: create databases mysql_db: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" db: "{{ item }}" state: present encoding: utf8 @@ -20,6 +23,10 @@ - name: create database users mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: "{{ item.user }}" password: "{{ item.pass }}" priv: "{{ item.db }}.*:ALL" @@ -29,6 +36,10 @@ - name: setup the migration db user mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: "{{ COMMON_MYSQL_MIGRATE_USER }}" password: "{{ COMMON_MYSQL_MIGRATE_PASS }}" priv: "{{ item }}.*:ALL" @@ -38,6 +49,10 @@ - name: create api user for the analytics api mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: "api001" password: "{{ ANALYTICS_API_DATABASES.default.PASSWORD }}" priv: '{{ ANALYTICS_API_DATABASES.default.NAME }}.*:ALL/reports.*:SELECT' @@ -45,6 +60,10 @@ - name: create read-only reports user for the analytics-api mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: reports001 password: "{{ ANALYTICS_API_DATABASES.reports.PASSWORD }}" priv: '{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:SELECT' @@ -52,6 +71,9 @@ - name: create a database for the hive metastore mysql_db: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" db: "{{ HIVE_METASTORE_DATABASE.name }}" state: "present" encoding: "latin1" @@ -59,6 +81,10 @@ - name: setup the edx-notes-api db user mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: "{{ EDX_NOTES_API_MYSQL_DB_USER }}" password: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}" priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE" @@ -66,6 +92,10 @@ - name: setup the read-only db user mysql_user: + login_host: "{{ EDXAPP_MYSQL_HOST }}" + login_user: "{{ COMMON_MYSQL_ADMIN_USER }}" + login_password: "{{ COMMON_MYSQL_ADMIN_PASS }}" + host: '%' name: "{{ COMMON_MYSQL_READ_ONLY_USER }}" password: "{{ COMMON_MYSQL_READ_ONLY_PASS }}" priv: "*.*:ALL" @@ -75,3 +105,5 @@ name: "{{ COMMON_MYSQL_ADMIN_USER }}" password: "{{ COMMON_MYSQL_ADMIN_PASS }}" priv: "*.*:CREATE USER" + when: EDXAPP_MYSQL_HOST == 'localhost' + From 8e333ab41865e326fee13200d959910dac5e02f3 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 13:59:23 +0900 Subject: [PATCH 31/60] Update native.sh changed: to cmscom checkout repo. --- util/install/native.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/install/native.sh b/util/install/native.sh index 97a466fc0ef..664409983b1 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -104,7 +104,9 @@ CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} cd /var/tmp git clone https://github.com/cmscom/configuration cd configuration -git checkout $CONFIGURATION_VERSION +#git checkout $CONFIGURATION_VERSION +#git checkout cmscom/ginkgo.master-terada-201808 +git checkout cmscom/${$OPENEDX_RELEASE2} git pull ## From 2835f32781adc8aeedf269350db89332517fdf93 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 14:02:02 +0900 Subject: [PATCH 32/60] Update native.sh bugfix: typo --- util/install/native.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/install/native.sh b/util/install/native.sh index 664409983b1..dbd68b837f3 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -105,8 +105,7 @@ cd /var/tmp git clone https://github.com/cmscom/configuration cd configuration #git checkout $CONFIGURATION_VERSION -#git checkout cmscom/ginkgo.master-terada-201808 -git checkout cmscom/${$OPENEDX_RELEASE2} +git checkout cmscom/ginkgo.master-terada-201808 git pull ## From 1466a6086d5131cdee8654bc810b16efa933cbe6 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:02:01 +0900 Subject: [PATCH 33/60] Update main.yml bugfix: added mongo host setting --- playbooks/roles/mongo_2_6/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/mongo_2_6/tasks/main.yml b/playbooks/roles/mongo_2_6/tasks/main.yml index a5d1c4a3c4e..c4de60a35a3 100644 --- a/playbooks/roles/mongo_2_6/tasks/main.yml +++ b/playbooks/roles/mongo_2_6/tasks/main.yml @@ -205,6 +205,7 @@ database: "{{ item.database }}" login_user: "{{ MONGO_ADMIN_USER }}" login_password: "{{ MONGO_ADMIN_PASSWORD }}" + login_host: "{{ MONGO_IP }}" name: "{{ item.user }}" password: "{{ item.password }}" roles: "{{ item.roles }}" @@ -220,6 +221,7 @@ database: "{{ item.database }}" login_user: "{{ MONGO_ADMIN_USER }}" login_password: "{{ MONGO_ADMIN_PASSWORD }}" + login_host: "{{ MONGO_IP }}" name: "{{ item.user }}" password: "{{ item.password }}" roles: "{{ item.roles }}" From 7532dd45f2fbdcba11279223b5e222c8b498a2f5 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:07:41 +0900 Subject: [PATCH 34/60] Update create_mongo_users.yml --- playbooks/edx-east/create_mongo_users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/edx-east/create_mongo_users.yml b/playbooks/edx-east/create_mongo_users.yml index 1188db18dc3..55c44fabd0f 100644 --- a/playbooks/edx-east/create_mongo_users.yml +++ b/playbooks/edx-east/create_mongo_users.yml @@ -35,7 +35,7 @@ database: "{{ item.database }}" login_user: "{{ MONGO_ADMIN_USER }}" login_password: "{{ MONGO_ADMIN_PASSWORD }}" - login_host: "{{ login_host }}" + login_host: "{{ MONGO_IP }}" name: "{{ item.user }}" password: "{{ item.password }}" roles: "{{ item.roles }}" From 393ab022448c8b3cb360e353241c224e20fdd39b Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:25:53 +0900 Subject: [PATCH 35/60] Update edx_cmscom.yml bugfix --- playbooks/edx_cmscom.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 69e69689ac9..278cb641e30 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -40,8 +40,9 @@ - role: edxremote - role: memcache when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" - - role: mongo - when: "'localhost' in EDXAPP_MONGO_HOSTS" +# - role: mongo +# when: "'localhost' in EDXAPP_MONGO_HOSTS" + - edx-east/create_mongo_users - role: rabbitmq rabbitmq_ip: 127.0.0.1 - role: edxapp From e967aba50b560c6b39985b3bde94bc6f25c9623f Mon Sep 17 00:00:00 2001 From: cmscom Date: Wed, 29 Aug 2018 23:44:34 -0700 Subject: [PATCH 36/60] added symlink --- playbooks/create_mongo_users.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 playbooks/create_mongo_users.yml diff --git a/playbooks/create_mongo_users.yml b/playbooks/create_mongo_users.yml new file mode 120000 index 00000000000..aabd6bff4e0 --- /dev/null +++ b/playbooks/create_mongo_users.yml @@ -0,0 +1 @@ +edx-east/create_mongo_users.yml \ No newline at end of file From 19999ab0566f8a22325ab99a100439d846f7ff08 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:48:49 +0900 Subject: [PATCH 37/60] Update edx_cmscom.yml bugfix --- playbooks/edx_cmscom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 278cb641e30..23bee421450 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -42,7 +42,7 @@ when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" # - role: mongo # when: "'localhost' in EDXAPP_MONGO_HOSTS" - - edx-east/create_mongo_users + - create_mongo_users - role: rabbitmq rabbitmq_ip: 127.0.0.1 - role: edxapp From 6be479a79451f558fa29d29a409f7a058d85b228 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:57:38 +0900 Subject: [PATCH 38/60] Update edx_cmscom.yml --- playbooks/edx_cmscom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 23bee421450..15d2b0a2e95 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -42,7 +42,7 @@ when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" # - role: mongo # when: "'localhost' in EDXAPP_MONGO_HOSTS" - - create_mongo_users + - import_tasks: edx-east/create_mongo_users.yml - role: rabbitmq rabbitmq_ip: 127.0.0.1 - role: edxapp From 06833d22329c334c6e9bad751ecbec882206374c Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 15:57:58 +0900 Subject: [PATCH 39/60] Delete create_mongo_users.yml --- playbooks/create_mongo_users.yml | 1 - 1 file changed, 1 deletion(-) delete mode 120000 playbooks/create_mongo_users.yml diff --git a/playbooks/create_mongo_users.yml b/playbooks/create_mongo_users.yml deleted file mode 120000 index aabd6bff4e0..00000000000 --- a/playbooks/create_mongo_users.yml +++ /dev/null @@ -1 +0,0 @@ -edx-east/create_mongo_users.yml \ No newline at end of file From ac928fcd8d5040d278b98b799237a71d1b109f37 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 16:00:13 +0900 Subject: [PATCH 40/60] Update edx_cmscom.yml --- playbooks/edx_cmscom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 15d2b0a2e95..01d2a8993e5 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -42,7 +42,7 @@ when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" # - role: mongo # when: "'localhost' in EDXAPP_MONGO_HOSTS" - - import_tasks: edx-east/create_mongo_users.yml + - include: edx-east/create_mongo_users.yml - role: rabbitmq rabbitmq_ip: 127.0.0.1 - role: edxapp From 1130bb74211b9bf3d4e6bde8bab2bca8ae815939 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 16:04:02 +0900 Subject: [PATCH 41/60] Update edx_cmscom.yml --- playbooks/edx_cmscom.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 01d2a8993e5..62fcc521830 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -11,6 +11,9 @@ roles: - python +# added mongo setting +- include: edx-east/create_mongo_users.yml + - name: Configure instance(s) hosts: all become: True @@ -42,7 +45,6 @@ when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" # - role: mongo # when: "'localhost' in EDXAPP_MONGO_HOSTS" - - include: edx-east/create_mongo_users.yml - role: rabbitmq rabbitmq_ip: 127.0.0.1 - role: edxapp From f0878bf4a2b7fe83aeb3c2a15115b9da897e74a2 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 16:15:18 +0900 Subject: [PATCH 42/60] Update create_mongo_users.yml add required mongo modules --- playbooks/edx-east/create_mongo_users.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/playbooks/edx-east/create_mongo_users.yml b/playbooks/edx-east/create_mongo_users.yml index 55c44fabd0f..58cf7d6da31 100644 --- a/playbooks/edx-east/create_mongo_users.yml +++ b/playbooks/edx-east/create_mongo_users.yml @@ -19,6 +19,16 @@ # login_host: 10.17.90.123 # repl_set: prod-edx-edxapp +- name: Install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ pymongo_version }}" +# extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - install + - install:base + - name: Create mongo users hosts: all gather_facts: False From f6ca1e35d8ae8ea21b3c7c38272f6917d99b0b0c Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 30 Aug 2018 16:19:13 +0900 Subject: [PATCH 43/60] Update create_mongo_users.yml --- playbooks/edx-east/create_mongo_users.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/playbooks/edx-east/create_mongo_users.yml b/playbooks/edx-east/create_mongo_users.yml index 58cf7d6da31..3e472000d08 100644 --- a/playbooks/edx-east/create_mongo_users.yml +++ b/playbooks/edx-east/create_mongo_users.yml @@ -19,16 +19,6 @@ # login_host: 10.17.90.123 # repl_set: prod-edx-edxapp -- name: Install python pymongo for mongo_user ansible module - pip: - name: pymongo - state: present - version: "{{ pymongo_version }}" -# extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" - tags: - - install - - install:base - - name: Create mongo users hosts: all gather_facts: False @@ -38,7 +28,7 @@ ansible_python_interpreter: "/usr/bin/env python" tasks: - name: install python mongo module - pip: name=pymongo state=present + pip: name=pymongo state=present version="{{ pymongo_version }}" - name: create a mongodb user mongodb_user: From 15b4cdafe09e17bdac57cbb883b537c8b2c3f1b3 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Fri, 12 Oct 2018 20:06:28 +0900 Subject: [PATCH 44/60] Update deploy.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit syncdbは不要なので削除 --- playbooks/roles/notifier/tasks/deploy.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/playbooks/roles/notifier/tasks/deploy.yml b/playbooks/roles/notifier/tasks/deploy.yml index 48cd2b699eb..c7755fca0c1 100644 --- a/playbooks/roles/notifier/tasks/deploy.yml +++ b/playbooks/roles/notifier/tasks/deploy.yml @@ -97,17 +97,18 @@ - "install" - "install:configuration" -- name: Syncdb - shell: "{{ NOTIFIER_VENV_DIR }}/bin/python manage.py syncdb" - args: - chdir: "{{ NOTIFIER_CODE_DIR }}" - become: true - become_user: "{{ notifier_user }}" - environment: "{{ notifier_env_vars }}" - when: migrate_db is defined and migrate_db|lower == "yes" - tags: - - "install" - - "install:configuration" +# 2018-10-12 +#- name: Syncdb +# shell: "{{ NOTIFIER_VENV_DIR }}/bin/python manage.py syncdb" +# args: +# chdir: "{{ NOTIFIER_CODE_DIR }}" +# become: true +# become_user: "{{ notifier_user }}" +# environment: "{{ notifier_env_vars }}" +# when: migrate_db is defined and migrate_db|lower == "yes" +# tags: +# - "install" +# - "install:configuration" - name: restart notifier-scheduler supervisorctl: From 1b52b5564ac662df1db3c654e20072c0ee7162b9 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 30 Oct 2018 17:04:17 +0900 Subject: [PATCH 45/60] Create analytics_single_cmscom.yml test commit --- playbooks/analytics_single_cmscom.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 playbooks/analytics_single_cmscom.yml diff --git a/playbooks/analytics_single_cmscom.yml b/playbooks/analytics_single_cmscom.yml new file mode 100644 index 00000000000..5fa21e6cf1b --- /dev/null +++ b/playbooks/analytics_single_cmscom.yml @@ -0,0 +1,22 @@ +# for insights +- name: Deploy all analytics services to a single node + hosts: all + become: True + gather_facts: True + vars: + migrate_db: "yes" + disable_edx_services: false + ENABLE_DATADOG: False + ENABLE_SPLUNKFORWARDER: False + ENABLE_NEWRELIC: False + roles: +# - aws +# - mysql +# - edxlocal + - memcache + - analytics_api + - analytics_pipeline + - role: nginx + nginx_sites: + - insights +- insights From 44123880fdf79b29ab512554d8af13f72a4317a9 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 30 Oct 2018 17:11:31 +0900 Subject: [PATCH 46/60] Create native-insights.sh test commit --- util/install/native-insights.sh | 135 ++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 util/install/native-insights.sh diff --git a/util/install/native-insights.sh b/util/install/native-insights.sh new file mode 100644 index 00000000000..d61a31e970e --- /dev/null +++ b/util/install/native-insights.sh @@ -0,0 +1,135 @@ +#!/bin/bash +## +## Installs the pre-requisites for running Open edX on a single Ubuntu 16.04 +## instance. This script is provided as a convenience and any of these +## steps could be executed manually. +## +## Note that this script requires that you have the ability to run +## commands as root via sudo. Caveat Emptor! +## + +## +## Sanity checks +## + +if [[ ! $OPENEDX_RELEASE ]]; then + echo "You must define OPENEDX_RELEASE" + exit +fi + +if [[ `lsb_release -rs` != "16.04" ]]; then + echo "This script is only known to work on Ubuntu 16.04, exiting..." + exit +fi + +## +## Log what's happening +## + +mkdir -p logs +log_file=logs/install-$(date +%Y%m%d-%H%M%S).log +exec > >(tee $log_file) 2>&1 +echo "Capturing output to $log_file" +echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')" + +function finish { + echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')" +} +trap finish EXIT + +echo "Installing release '$OPENEDX_RELEASE'" + +## +## Set ppa repository source for gcc/g++ 4.8 in order to install insights properly +## +sudo apt-get install -y python-software-properties +sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + +## +## Update and Upgrade apt packages +## +sudo apt-get update -y +sudo apt-get upgrade -y + +## added require modules. @cmscom +sudo apt-get install -y mariadb-client + +## +## Install system pre-requisites +## +sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ +sudo pip install --upgrade pip==9.0.3 +sudo pip install --upgrade setuptools==39.0.1 +sudo -H pip install --upgrade virtualenv==15.2.0 + +## +## Overridable version variables in the playbooks. Each can be overridden +## individually, or with $OPENEDX_RELEASE. +## +VERSION_VARS=( + edx_platform_version + certs_version + forum_version + xqueue_version + configuration_version + demo_version + NOTIFIER_VERSION + INSIGHTS_VERSION + ANALYTICS_API_VERSION + ECOMMERCE_VERSION + ECOMMERCE_WORKER_VERSION + DISCOVERY_VERSION +) + +for var in ${VERSION_VARS[@]}; do + # Each variable can be overridden by a similarly-named environment variable, + # or OPENEDX_RELEASE, if provided. + ENV_VAR=$(echo $var | tr '[:lower:]' '[:upper:]') + eval override=\${$ENV_VAR-\$OPENEDX_RELEASE} + if [ -n "$override" ]; then + EXTRA_VARS="-e $var=$override $EXTRA_VARS" + fi +done + +# my-passwords.yml is the file made by generate-passwords.sh. +if [[ -f my-passwords.yml ]]; then + EXTRA_VARS="-e@$(pwd)/my-passwords.yml $EXTRA_VARS" +fi + +CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} + +## +## Clone the configuration repository and run Ansible +## +cd /var/tmp +git clone https://github.com/cmscom/configuration +cd configuration +#git checkout $CONFIGURATION_VERSION +git checkout cmscom/ginkgo.master-terada-201808 +git pull + +## +## Install the ansible requirements +## +cd /var/tmp/configuration +sudo -H pip install -r requirements.txt + +## +## Run the edx_sandbox.yml playbook in the configuration/playbooks directory +## +cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./analytics_single_cmscom.yml -i "localhost," $EXTRA_VARS "$@" +ansible_status=$? + +if [[ $ansible_status -ne 0 ]]; then + echo " " + echo "========================================" + echo "Ansible failed!" + echo "----------------------------------------" + echo "If you need help, see https://open.edx.org/getting-help ." + echo "When asking for help, please provide as much information as you can." + echo "These might be helpful:" + echo " Your log file is at $log_file" + echo " Your environment:" + env | egrep -i 'version|release' | sed -e 's/^/ /' + echo "========================================" +fi From 583fc8d5109a7a5181528cff7bc28a8a8a2cba4d Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 30 Oct 2018 17:23:04 +0900 Subject: [PATCH 47/60] Update analytics_single_cmscom.yml fixed typo --- playbooks/analytics_single_cmscom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/analytics_single_cmscom.yml b/playbooks/analytics_single_cmscom.yml index 5fa21e6cf1b..ea44904b021 100644 --- a/playbooks/analytics_single_cmscom.yml +++ b/playbooks/analytics_single_cmscom.yml @@ -19,4 +19,4 @@ - role: nginx nginx_sites: - insights -- insights + - insights From c09c41b44c0dcad8680a0623c17cc770d56dad90 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 30 Oct 2018 18:45:32 +0900 Subject: [PATCH 48/60] Update main.yml change download url --- playbooks/roles/sqoop/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/sqoop/defaults/main.yml b/playbooks/roles/sqoop/defaults/main.yml index 0f44dd93e07..b982c4a0616 100644 --- a/playbooks/roles/sqoop/defaults/main.yml +++ b/playbooks/roles/sqoop/defaults/main.yml @@ -28,7 +28,8 @@ sqoop_temporary_dir: /var/tmp sqoop_base_filename: "sqoop-{{ SQOOP_VERSION }}.bin__hadoop-{{ SQOOP_HADOOP_VERSION }}" sqoop_dist: filename: "{{ sqoop_base_filename }}.tar.gz" - url: "http://www.apache.org/dist/sqoop/{{ SQOOP_VERSION }}/{{ sqoop_base_filename }}.tar.gz" +# url: "http://www.apache.org/dist/sqoop/{{ SQOOP_VERSION }}/{{ sqoop_base_filename }}.tar.gz" + url: "http://archive.apache.org/dist/sqoop/{{ SQOOP_VERSION }}/{{ sqoop_base_filename }}.tar.gz" sha256sum: d582e7968c24ff040365ec49764531cb76dfa22c38add5f57a16a57e70d5d496 sqoop_mysql_connector_dist: filename: "mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz" From 7a2db7c050736b35e12459f062c1e44556addfce Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 1 Nov 2018 19:08:29 +0900 Subject: [PATCH 49/60] Update main.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TODO: MySQLの設定変更部分は不要と判断した --- .../roles/analytics_pipeline/tasks/main.yml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/playbooks/roles/analytics_pipeline/tasks/main.yml b/playbooks/roles/analytics_pipeline/tasks/main.yml index c7d8da70c13..cc1b8d9d869 100644 --- a/playbooks/roles/analytics_pipeline/tasks/main.yml +++ b/playbooks/roles/analytics_pipeline/tasks/main.yml @@ -220,15 +220,17 @@ - install - install:configuration -- name: Grant access to table storing test data in output database - mysql_user: - user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.username }}" - password: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.password }}" - priv: 'acceptance%.*:ALL' - append_privs: yes - tags: - - install - - install:configuration +# TODO: すでに設定されているデータ。おそらく不要なはず +#- name: Grant access to table storing test data in output database +# mysql_user: +# user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.username }}" +# password: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.password }}" +# host: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.host }}" +# priv: 'acceptance%.*:ALL' +# append_privs: yes +# tags: +# - install +# - install:configuration - name: Test if Hive metadata store schema exists shell: ". {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && {{ HIVE_HOME }}/bin/hive | tr '\n' ' '" From bc3f194eb4d464c154c1539c876c326f07a11007 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 21 Feb 2019 17:33:25 +0900 Subject: [PATCH 50/60] Update edx_cmscom.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modified: 検証環境用にインストールを改変 --- playbooks/edx_cmscom.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 62fcc521830..c3634f1593b 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -12,7 +12,7 @@ - python # added mongo setting -- include: edx-east/create_mongo_users.yml +#- include: edx-east/create_mongo_users.yml - name: Configure instance(s) hosts: all @@ -40,7 +40,7 @@ - lms # - role: edxlocal # when: EDXAPP_MYSQL_HOST == 'localhost' - - role: edxremote +# - role: edxremote - role: memcache when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" # - role: mongo @@ -56,7 +56,7 @@ - analytics_api # - insights # not ready yet: - edx_notes_api - - demo +# - demo - oauth_client_setup - oraclejdk - role: elasticsearch From 43d726530c2d1bb17dd83c160463f0cfdb92774b Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Tue, 26 Feb 2019 14:10:52 +0900 Subject: [PATCH 51/60] Update main.yml bugfix: rubygems version --- playbooks/roles/rbenv/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/rbenv/tasks/main.yml b/playbooks/roles/rbenv/tasks/main.yml index 9ec5cc16363..47b474257d5 100644 --- a/playbooks/roles/rbenv/tasks/main.yml +++ b/playbooks/roles/rbenv/tasks/main.yml @@ -188,7 +188,7 @@ - install:base - name: update rubygems - shell: "gem install rubygems-update && update_rubygems" + shell: "gem install rubygems-update -v '<3' && update_rubygems" become_user: "{{ rbenv_user }}" environment: "{{ rbenv_environment }}" tags: From 188c73267c6d066e8b8ad760b78fcede631d74c3 Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Mon, 4 Mar 2019 12:03:02 +0900 Subject: [PATCH 52/60] Update deploy.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bugfix: この処理がなくなっているのでコメントアウト（テスト） --- playbooks/roles/notifier/tasks/deploy.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/notifier/tasks/deploy.yml b/playbooks/roles/notifier/tasks/deploy.yml index c7755fca0c1..11863c97ce8 100644 --- a/playbooks/roles/notifier/tasks/deploy.yml +++ b/playbooks/roles/notifier/tasks/deploy.yml @@ -121,13 +121,14 @@ - "manage" - "manage:start" -- name: restart notifier-celery-workers - supervisorctl: - name: "notifier-celery-workers" - state: restarted - config: "{{ supervisor_cfg }}" - supervisorctl_path: "{{ supervisor_ctl }}" - when: not disable_edx_services - tags: - - "manage" - - "manage:start" +# cmscom@2019-03-04 +#- name: restart notifier-celery-workers +# supervisorctl: +# name: "notifier-celery-workers" +# state: restarted +# config: "{{ supervisor_cfg }}" +# supervisorctl_path: "{{ supervisor_ctl }}" +# when: not disable_edx_services +# tags: +# - "manage" +# - "manage:start" From 72aec1a6193a166091c9d3e9a94f32366d29f93e Mon Sep 17 00:00:00 2001 From: Haru TAZAWA Date: Thu, 14 Mar 2019 19:05:08 +0900 Subject: [PATCH 53/60] Update analytics_single_cmscom.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lms/cms側でおそらく一緒にマイグレーション済のため変更 --- playbooks/analytics_single_cmscom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/analytics_single_cmscom.yml b/playbooks/analytics_single_cmscom.yml index ea44904b021..038d9863abf 100644 --- a/playbooks/analytics_single_cmscom.yml +++ b/playbooks/analytics_single_cmscom.yml @@ -4,7 +4,7 @@ become: True gather_facts: True vars: - migrate_db: "yes" + migrate_db: "no" disable_edx_services: false ENABLE_DATADOG: False ENABLE_SPLUNKFORWARDER: False From ca822211c7628cadf09ecd8fdb9a60548e86c453 Mon Sep 17 00:00:00 2001 From: terapyon Date: Thu, 4 Jul 2019 12:17:10 +0900 Subject: [PATCH 54/60] forgot adding files --- playbooks/roles/edxapp/defaults/main.yml | 6 ------ playbooks/roles/forum/defaults/main.yml | 4 ---- util/vpc-tools/abbey.py | 3 --- 3 files changed, 13 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 5ac88bbd254..379333559ed 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -809,7 +809,6 @@ EDXAPP_VIDEO_IMAGE_SETTINGS: location: "{{ edxapp_media_dir }}/" base_url: "{{ EDXAPP_MEDIA_URL }}/" DIRECTORY_PREFIX: 'video-images/' -<<<<<<< HEAD EDXAPP_VIDEO_TRANSCRIPTS_MAX_AGE: 31536000 @@ -820,8 +819,6 @@ EDXAPP_VIDEO_TRANSCRIPTS_SETTINGS: location: "{{ edxapp_media_dir }}/" base_url: "{{ EDXAPP_MEDIA_URL }}/" DIRECTORY_PREFIX: 'video-transcripts/' -======= ->>>>>>> cmscom/ginkgo.master # Course Block Structures EDXAPP_BLOCK_STRUCTURES_SETTINGS: @@ -1556,7 +1553,6 @@ SERVICE_WORKER_USERS: username: "{{ EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME }}" is_staff: true is_superuser: false -<<<<<<< HEAD - email: "{{ EDXAPP_VEDA_SERVICE_USER_EMAIL }}" username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME }}" is_staff: true @@ -1580,5 +1576,3 @@ SERVICE_WORKER_USERS: is_superuser: false EDXAPP_ENABLE_DJANGO_ADMIN_RESTRICTION: false -======= ->>>>>>> cmscom/ginkgo.master diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index e877a001e54..7c739722ff5 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -48,11 +48,7 @@ FORUM_RESTART_DELAY: 60 # Set to rebuild the forum ElasticSearch index from the database. FORUM_REBUILD_INDEX: false -<<<<<<< HEAD forum_base_env: &forum_base_env -======= -forum_environment: ->>>>>>> cmscom/ginkgo.master RBENV_ROOT: "{{ forum_rbenv_root }}" GEM_HOME: "{{ forum_gem_root }}" GEM_PATH: "{{ forum_gem_root }}" diff --git a/util/vpc-tools/abbey.py b/util/vpc-tools/abbey.py index c7d2345aa6c..6ef52588d16 100644 --- a/util/vpc-tools/abbey.py +++ b/util/vpc-tools/abbey.py @@ -330,10 +330,7 @@ def create_instance_args(): VIRTUAL_ENV_VERSION="15.2.0" PIP_VERSION="9.0.3" SETUPTOOLS_VERSION="39.0.1" -<<<<<<< HEAD -======= EDX_PPA="deb http://ppa.edx.org precise main" ->>>>>>> cmscom/ginkgo.master EDX_PPA_KEY_SERVER="keyserver.ubuntu.com" EDX_PPA_KEY_ID="B41E5E3969464050" From b47eea7003484b7a990a89b2378acbc15050514d Mon Sep 17 00:00:00 2001 From: ringobouya Date: Fri, 19 Jul 2019 18:24:09 +0900 Subject: [PATCH 55/60] Update edx_cmscom.yml updated from hawthorn version (edx_sandbox.yml). --- playbooks/edx_cmscom.yml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index c3634f1593b..768e9ed1625 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -11,6 +11,7 @@ roles: - python +# TODO: 2019-07-19 # added mongo setting #- include: edx-east/create_mongo_users.yml @@ -25,6 +26,14 @@ # Set to false if deployed behind another proxy/load balancer. NGINX_SET_X_FORWARDED_HEADERS: True DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}' + ecommerce_create_demo_data: false + credentials_create_demo_data: false + SANDBOX_ENABLE_DISCOVERY: true + SANDBOX_ENABLE_ECOMMERCE: false + SANDBOX_ENABLE_ANALYTICS_API: true + SANDBOX_ENABLE_INSIGHTS: false + SANDBOX_ENABLE_RABBITMQ: true + JOURNALS_ENABLED: false roles: - role: swapfile SWAPFILE_SIZE: 4GB @@ -35,7 +44,6 @@ - lms - forum - xqueue -# - ecommerce nginx_default_sites: - lms # - role: edxlocal @@ -43,18 +51,23 @@ # - role: edxremote - role: memcache when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" -# - role: mongo +# - role: mongo_3_2 # when: "'localhost' in EDXAPP_MONGO_HOSTS" - role: rabbitmq rabbitmq_ip: 127.0.0.1 + when: SANDBOX_ENABLE_RABBITMQ - role: edxapp celery_worker: True - edxapp -# - ecommerce -# - role: ecomworker -# ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 - - analytics_api -# - insights + - role: ecommerce + when: SANDBOX_ENABLE_ECOMMERCE + - role: ecomworker + ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1 + when: SANDBOX_ENABLE_ECOMMERCE + - role: analytics_api + when: SANDBOX_ENABLE_ANALYTICS_API + - role: insights + when: SANDBOX_ENABLE_INSIGHTS # not ready yet: - edx_notes_api # - demo - oauth_client_setup @@ -62,7 +75,10 @@ - role: elasticsearch when: "'localhost' in EDXAPP_ELASTIC_SEARCH_CONFIG|map(attribute='host')" - forum - - discovery + - role: discovery + when: SANDBOX_ENABLE_DISCOVERY + - role: journals + when: JOURNALS_ENABLED - role: notifier NOTIFIER_DIGEST_TASK_INTERVAL: 5 - role: xqueue @@ -73,7 +89,7 @@ when: COMMON_ENABLE_DATADOG - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER - - role: newrelic - when: COMMON_ENABLE_NEWRELIC - role: postfix_queue when: POSTFIX_QUEUE_EXTERNAL_SMTP_HOST != '' + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG From 77ccb2db98ec4cd014f3dd8f18d4de52cda04ba5 Mon Sep 17 00:00:00 2001 From: ringobouya Date: Fri, 19 Jul 2019 18:36:30 +0900 Subject: [PATCH 56/60] Create env_hawthorn added --- util/install/env_hawthorn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 util/install/env_hawthorn diff --git a/util/install/env_hawthorn b/util/install/env_hawthorn new file mode 100644 index 00000000000..c3480718a8e --- /dev/null +++ b/util/install/env_hawthorn @@ -0,0 +1,3 @@ +#!/bin/sh +export OPENEDX_RELEASE=open-release/hawthorn.master +export OPENEDX_RELEASE2=cmscom/hawthorn.master-from-ginkgo From 72bc377687f85622aba719c0c68a0a7c53416aba Mon Sep 17 00:00:00 2001 From: ringobouya Date: Fri, 19 Jul 2019 18:44:18 +0900 Subject: [PATCH 57/60] Update native.sh updated checkout branch --- util/install/native.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/install/native.sh b/util/install/native.sh index 1e775efd19d..266c905a9fa 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -105,9 +105,8 @@ CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} cd /var/tmp git clone https://github.com/cmscom/configuration cd configuration -git checkout $CONFIGURATION_VERSION #git checkout $CONFIGURATION_VERSION -#git checkout cmscom/ginkgo.master-terada-201808 +git checkout cmscom/hawthorn.master-from-ginkgo git pull ## From 06a41873a71caeea6f4c84ba2589e562aedd09b6 Mon Sep 17 00:00:00 2001 From: ringobouya Date: Tue, 6 Aug 2019 18:17:46 +0900 Subject: [PATCH 58/60] Update generate-passwords.sh --- util/install/generate-passwords.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install/generate-passwords.sh b/util/install/generate-passwords.sh index 0b3055220df..ca748d8d466 100755 --- a/util/install/generate-passwords.sh +++ b/util/install/generate-passwords.sh @@ -4,7 +4,7 @@ # a new file just like it, with the generated values. #TARGET=${OPENEDX_RELEASE-master} -TARGET=${OPENEDX_RELEASE} +TARGET=${OPENEDX_RELEASE2} wget -q "https://raw.githubusercontent.com/cmscom/configuration/$TARGET/playbooks/sample_vars/passwords.yml" -O passwords-template.yml while IFS= read -r line; do From 62cd28d53409f1cd36e6c02877c774bec2a7a82b Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Thu, 19 Dec 2019 17:55:25 +0900 Subject: [PATCH 59/60] use edx local --- playbooks/edx_cmscom.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index 768e9ed1625..edb562e1e11 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -34,6 +34,8 @@ SANDBOX_ENABLE_INSIGHTS: false SANDBOX_ENABLE_RABBITMQ: true JOURNALS_ENABLED: false + # test 20191119 + EDXAPP_MYSQL_HOST: localhost roles: - role: swapfile SWAPFILE_SIZE: 4GB @@ -46,8 +48,9 @@ - xqueue nginx_default_sites: - lms -# - role: edxlocal -# when: EDXAPP_MYSQL_HOST == 'localhost' +# test 20191119 off to on + - role: edxlocal + when: EDXAPP_MYSQL_HOST == 'localhost' # - role: edxremote - role: memcache when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" From bfb6fb684817a5839a1ea4fb37e68723d3e0cdc8 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Fri, 20 Dec 2019 17:21:56 +0900 Subject: [PATCH 60/60] Install mongo for local --- playbooks/edx_cmscom.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/edx_cmscom.yml b/playbooks/edx_cmscom.yml index edb562e1e11..f0db289e6dc 100644 --- a/playbooks/edx_cmscom.yml +++ b/playbooks/edx_cmscom.yml @@ -54,8 +54,11 @@ # - role: edxremote - role: memcache when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" +# 20191220 comment in for local install # - role: mongo_3_2 # when: "'localhost' in EDXAPP_MONGO_HOSTS" + - role: mongo_3_2 + when: "'localhost' in EDXAPP_MONGO_HOSTS" - role: rabbitmq rabbitmq_ip: 127.0.0.1 when: SANDBOX_ENABLE_RABBITMQ