PLAT-2060 Use pip-tools to manage requirements files

Makefile

@@ -1,9 +1,14 @@
 # Do things in edx-platform
+.PHONY: clean extract_translations help pull_translations push_translations requirements upgrade
 
 # Careful with mktemp syntax: it has to work on Mac and Ubuntu, which have differences.
 PRIVATE_FILES := $(shell mktemp -u /tmp/private_files.XXXXXX)
 
-clean:
+help: ## display this help message
+	@echo "Please use \`make <target>' where <target> is one of"
+	@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | sort | awk -F ':.*?## ' 'NF==2 {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+clean: ## archive and delete most git-ignored files
 	# Remove all the git-ignored stuff, but save and restore things marked
 	# by start-noclean/end-noclean. Include Makefile in the tarball so that
 	# there's always at least one file even if there are no private files.
@@ -13,16 +18,13 @@ clean:
 	tar xf $(PRIVATE_FILES)
 	rm $(PRIVATE_FILES)
 
-extract_translations:
-	# Extract localizable strings from sources
+extract_translations: ## extract localizable strings from sources
 	i18n_tool extract -vv
 
-push_translations:
-	# Push source strings to Transifex for translation
+push_translations: ## push source strings to Transifex for translation
 	i18n_tool transifex push
 
-pull_translations:
-	## Pull translations from Transifex
+pull_translations: ## pull translations from Transifex
 	git clean -fdX conf/locale
 	i18n_tool transifex pull
 	i18n_tool extract
@@ -32,3 +34,32 @@ pull_translations:
 	git clean -fdX conf/locale/rtl
 	git clean -fdX conf/locale/eo
 	i18n_tool validate
+
+requirements: ## install development environment requirements
+	pip install -qr requirements/edx/development.txt --exists-action w
+
+upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints
+	pip install -qr requirements/edx/pip-tools.txt
+	# Make sure to compile files after any other files they include!
+	pip-compile --upgrade -o requirements/edx/pip-tools.txt requirements/edx/pip-tools.in
+	pip-compile --upgrade -o requirements/edx/coverage.txt requirements/edx/coverage.in
+	pip-compile --upgrade -o requirements/edx/paver.txt requirements/edx/paver.in
+	pip-compile --upgrade -o requirements/edx-sandbox/shared.txt requirements/edx-sandbox/shared.in
+	pip-compile --upgrade -o requirements/edx-sandbox/base.txt requirements/edx-sandbox/base.in
+	pip-compile --upgrade -o requirements/edx/base.txt requirements/edx/base.in
+	pip-compile --upgrade -o requirements/edx/testing.txt requirements/edx/testing.in
+	pip-compile --upgrade -o requirements/edx/development.txt requirements/edx/development.in
+	# Post process all of the files generated above to work around open pip-tools issues
+	scripts/post-pip-compile.sh \
+        requirements/edx/pip-tools.txt \
+	    requirements/edx/coverage.txt \
+	    requirements/edx/paver.txt \
+	    requirements/edx-sandbox/shared.txt \
+	    requirements/edx-sandbox/base.txt \
+	    requirements/edx/base.txt \
+	    requirements/edx/testing.txt \
+	    requirements/edx/development.txt
+	# Let tox control the Django version for tests
+	grep "^django==" requirements/edx/base.txt > requirements/edx/django.txt
+	sed '/^[dD]jango==/d' requirements/edx/testing.txt > requirements/edx/testing.tmp
+	mv requirements/edx/testing.tmp requirements/edx/testing.txt

circle.yml

@@ -12,26 +12,14 @@ dependencies:
     - npm install
 
     - pip install setuptools
-    - pip install --exists-action w -r requirements/edx/paver.txt
 
     # Mirror what paver install_prereqs does.
     # After a successful build, CircleCI will
     # cache the virtualenv at that state, so that
     # the next build will not need to install them
     # from scratch again.
     - pip install --exists-action w -r requirements/edx/pre.txt
-    - pip install --exists-action w -r requirements/edx/github.txt
-    - pip install --exists-action w -r requirements/edx/local.txt
-
-    # HACK: within base.txt stevedore had a
-    # dependency on a version range of pbr.
-    # Install a version which falls within that range.
-    - pip install  --exists-action w pbr==0.9.0
-    - pip install --exists-action w -r requirements/edx/django.txt
-    - pip install --exists-action w -r requirements/edx/base.txt
-    - pip install --exists-action w -r requirements/edx/paver.txt
     - pip install --exists-action w -r requirements/edx/testing.txt
-    - if [ -e requirements/edx/post.txt ]; then pip install --exists-action w -r requirements/edx/post.txt ; fi
 
     - pip install coveralls==1.0
 

common/lib/capa/capa/safe_exec/README.rst

@@ -19,8 +19,6 @@ __ https://github.com/edx/codejail/blob/master/README.rst
    need to install the requirements from requirements/edx-sandbox::
 
     $ pip install -r requirements/edx-sandbox/base.txt
-    $ pip install -r requirements/edx-sandbox/local.txt
-    $ pip install -r requirements/edx-sandbox/post.txt
 
 2. At the instruction to create the AppArmor profile, you'll need a line in
    the profile for the sandbox packages.  <EDXPLATFORM> is the full path to

common/lib/chem/setup.py

@@ -8,6 +8,6 @@
         "pyparsing==2.2.0",
         "numpy==1.6.2",
         "scipy==0.14.0",
-        "nltk==3.2.5",
+        "nltk",
     ],
 )

openedx.yaml

@@ -3,5 +3,5 @@
 
 nick: edx
 oeps: {}
-openedx-release: {ref: release, requirements: requirements/edx/github.txt}
+openedx-release: {ref: release}
 track-pulls: true

openedx/core/djangoapps/user_api/tests/test_views.py

@@ -2590,7 +2590,8 @@ def _assert_time_zone_is_valid(self, time_zone_info):
         self.assertIn(time_zone_name, common_timezones_set)
         self.assertEqual(time_zone_info['description'], get_display_time_zone(time_zone_name))
 
-    @ddt.data((ALL_TIME_ZONES_URI, 436),
+    # The time zones count may need to change each time we upgrade pytz
+    @ddt.data((ALL_TIME_ZONES_URI, 439),
               (COUNTRY_TIME_ZONES_URI, 28))
     @ddt.unpack
     def test_get_basic(self, country_uri, expected_count):

pavelib/prereqs.py

@@ -21,20 +21,10 @@
 # If you make any changes to this list you also need to make
 # a corresponding change to circle.yml, which is how the python
 # prerequisites are installed for builds on circleci.com
-PYTHON_REQ_FILES = [
-    'requirements/edx/pre.txt',
-    'requirements/edx/github.txt',
-    'requirements/edx/local.txt',
-    'requirements/edx/django.txt',
-    'requirements/edx/base.txt',
-    'requirements/edx/paver.txt',
-    'requirements/edx/development.txt',
-    'requirements/edx/testing.txt',
-    'requirements/edx/post.txt',
-]
 if 'TOXENV' in os.environ:
-    # Let tox manage the Django version
-    PYTHON_REQ_FILES.remove('requirements/edx/django.txt')
+    PYTHON_REQ_FILES = ['requirements/edx/testing.txt']
+else:
+    PYTHON_REQ_FILES = ['requirements/edx/development.txt']
 
 # Developers can have private requirements, for local copies of github repos,
 # or favorite debugging tools, etc.

pavement.py

@@ -6,7 +6,7 @@
 # In local dev, we usually don't need to do this, because Python
 # automatically puts the current working directory on the system path.
 # In Jenkins, however, we have multiple copies of the edx-platform repo,
-# each of which run "pip install -e ." (as part of requirements/edx/local.txt)
+# each of which run "pip install -e ." (as part of requirements/edx/local.in)
 # Until we re-run pip install, the other copies of edx-platform could
 # take precedence, leading to some very strange results.
 sys.path.insert(0, os.path.dirname(__file__))

requirements/edx-sandbox/base.in

@@ -0,0 +1,11 @@
+# Packages to install in the Python sandbox for secured execution of code provided by course authors.
+#
+# DON'T JUST ADD NEW DEPENDENCIES!!!
+#
+# If you open a pull request that adds a new dependency, you should:
+#   * verify that the dependency has a license compatible with AGPLv3
+#   * confirm that it has no system requirements beyond what we already install
+#   * run "make upgrade" to update the detailed requirements files
+
+-r shared.txt                       # Dependencies in common with LMS and Studio
+matplotlib==1.3.1                   # 2D plotting library

requirements/edx-sandbox/base.txt

@@ -1,16 +1,33 @@
-# DON'T JUST ADD NEW DEPENDENCIES!!!
 #
-# If you open a pull request that adds a new dependency, you should notify:
-#   * @edx/ospr - to check licensing
-#   * @edx/devops - to check system requirements
-
-# Pin packaging tools the same as edxapp.  Keep them in sync for our sanity.
-setuptools==37.0.0
-pip==9.0.1
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 
+-e common/lib/calc
+-e common/lib/chem
+-e common/lib/sandbox-packages
+-e common/lib/symmath
+asn1crypto==0.24.0
+backports-abc==0.5        # via tornado
+cffi==1.11.5
+cryptography==2.1.4
+enum34==1.1.6
+futures==3.2.0            # via tornado
+idna==2.6
+ipaddress==1.0.19
+lxml==3.8.0
+matplotlib==1.3.1
+networkx==1.7
 nltk==3.2.5
+nose==1.3.7               # via matplotlib
 numpy==1.6.2
-networkx==1.7
+pycparser==2.18
+pyparsing==2.0.7
+python-dateutil==2.7.2    # via matplotlib
+scipy==0.14.0
+singledispatch==3.4.0.3   # via tornado
+six==1.11.0
 sympy==0.7.1
-pyparsing==2.2.0
-cryptography==1.9
+tornado==5.0.2            # via matplotlib

requirements/edx-sandbox/local.txt

@@ -1,13 +1,3 @@
-# DON'T JUST ADD NEW DEPENDENCIES!!!
-#
-# If you open a pull request that adds a new dependency, you should notify:
-#   * @edx/ospr - to check licensing
-#   * @edx/devops - to check system requirements
-
-# Install these packages from the edx-platform working tree
-# NOTE: if you change code in these packages, you MUST change the version
-# number in its setup.py or the code WILL NOT be installed during deploy.
-common/lib/calc
-common/lib/chem
-common/lib/sandbox-packages
-common/lib/symmath
+# Placeholder for code which hasn't yet been updated to no longer use this file.
+# The requirements which were here have been moved to shared.in, and are now
+# incorporated into base.txt

requirements/edx-sandbox/post.txt

@@ -1,11 +1,3 @@
-# DON'T JUST ADD NEW DEPENDENCIES!!!
-#
-# If you open a pull request that adds a new dependency, you should notify:
-#   * @edx/ospr - to check licensing
-#   * @edx/devops - to check system requirements
-
-# Packages to install in the Python sandbox for secured execution.
-scipy==0.14.0
-# lxml is also in requirements/edx/base.txt
-lxml==3.8.0
-matplotlib==1.3.1
+# Placeholder for code which hasn't yet been updated to no longer use this file.
+# The requirements which were here have been moved to shared.in and base.in,
+# and are now incorporated into base.txt

requirements/edx-sandbox/shared.in

@@ -0,0 +1,25 @@
+# Core dependencies shared between Python sandboxes for secured execution and edx-platform.
+#
+# DON'T JUST ADD NEW DEPENDENCIES!!!
+#
+# If you open a pull request that adds a new dependency, you should:
+#   * verify that the dependency has a license compatible with AGPLv3
+#   * confirm that it has no system requirements beyond what we already install
+#   * run "make upgrade" to update the detailed requirements files
+
+cryptography==2.1.4                 # Implementations of assorted cryptography algorithms
+lxml==3.8.0                         # XML parser
+networkx==1.7                       # Utilities for creating, manipulating, and studying network graphs
+nltk                                # Natural language processing; used by the chem package
+numpy==1.6.2                        # Numeric array processing utilities; used by calc, chem, and scipy
+pyparsing                           # Python parsing library; used by the calc package
+sympy==0.7.1                        # Symbolic math library; used by the calc package
+scipy==0.14.0                       # Math, science, and engineering library; used by the calc package
+
+# Install these packages from the edx-platform working tree
+# NOTE: if you change code in these packages, you MUST change the version
+# number in its setup.py or the code WILL NOT be installed during deploy.
+-e common/lib/calc
+-e common/lib/chem
+-e common/lib/sandbox-packages
+-e common/lib/symmath

requirements/edx-sandbox/shared.txt

@@ -0,0 +1,26 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+
+-e common/lib/calc
+-e common/lib/chem
+-e common/lib/sandbox-packages
+-e common/lib/symmath
+asn1crypto==0.24.0        # via cryptography
+cffi==1.11.5              # via cryptography
+cryptography==2.1.4
+enum34==1.1.6             # via cryptography
+idna==2.6                 # via cryptography
+ipaddress==1.0.19         # via cryptography
+lxml==3.8.0
+networkx==1.7
+nltk==3.2.5
+numpy==1.6.2
+pycparser==2.18           # via cffi
+pyparsing==2.0.7
+scipy==0.14.0
+six==1.11.0               # via cryptography, nltk
+sympy==0.7.1