Don't require pygraphviz for regenerate.sh (#17485)

Summary:
closes #17336

Do not overwrite config.yml if script throws an error
Pull Request resolved: https://github.com/pytorch/pytorch/pull/17485

Differential Revision: D14604388

Pulled By: kostmo

fbshipit-source-id: 5024545e3a8711abdbc0800911c766929dbca196

24 files changed, 252 insertions, 255 deletions

diff --git a/.circleci/README.md b/.circleci/README.md
index 74ac300ea9..7a70cb9c21 100644
--- a/.circleci/README.md
+++ b/.circleci/README.md
@@ -23,7 +23,9 @@ The documentation, in the form of diagrams, is automatically generated and canno
 Furthermore, consistency is enforced within the YAML config itself, by using a single source of data to generate
 multiple parts of the file.
 
-See https://github.com/pytorch/pytorch/issues/17038
+* Facilitates one-off culling/enabling of CI configs for testing PRs on special targets
+
+Also see https://github.com/pytorch/pytorch/issues/17038
 
 
 Future direction
diff --git a/.circleci/cimodel/data/__init__.py b/.circleci/cimodel/data/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/.circleci/cimodel/data/__init__.py
diff --git a/.circleci/cimodel/make_build_configs.py b/.circleci/cimodel/data/binary_build_data.py
index 8782d77373..1e5c563bf4 100644
--- a/.circleci/cimodel/make_build_configs.py
+++ b/.circleci/cimodel/data/binary_build_data.py
@@ -14,7 +14,8 @@ to produce a visualization of config dimensions.
 
 from collections import OrderedDict
 
-from cimodel.conf_tree import ConfigNode
+from cimodel.lib.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
 
 
 LINKING_DIMENSIONS = [
@@ -32,23 +33,8 @@ def get_processor_arch_name(cuda_version):
     return "cpu" if not cuda_version else "cu" + cuda_version
 
 
-CUDA_VERSIONS = [
-    None,  # cpu build
-    "80",
-    "90",
-    "100",
-]
-
-STANDARD_PYTHON_VERSIONS = [
-    "2.7",
-    "3.5",
-    "3.6",
-    "3.7",
-]
-
-
 CONFIG_TREE_DATA = OrderedDict(
-    linux=(CUDA_VERSIONS, OrderedDict(
+    linux=(dimensions.CUDA_VERSIONS, OrderedDict(
         manywheel=[
             "2.7m",
             "2.7mu",
@@ -56,14 +42,14 @@ CONFIG_TREE_DATA = OrderedDict(
             "3.6m",
             "3.7m",
         ],
-        conda=STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.STANDARD_PYTHON_VERSIONS,
         libtorch=[
             "2.7m",
         ]
     )),
     macos=([None], OrderedDict(
-        wheel=STANDARD_PYTHON_VERSIONS,
-        conda=STANDARD_PYTHON_VERSIONS,
+        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.STANDARD_PYTHON_VERSIONS,
         libtorch=[
             "2.7",
         ],
diff --git a/.circleci/cimodel/binary_build_definitions.py b/.circleci/cimodel/data/binary_build_definitions.py
index 547041aa1d..161d6a9291 100644
--- a/.circleci/cimodel/binary_build_definitions.py
+++ b/.circleci/cimodel/data/binary_build_definitions.py
@@ -2,10 +2,10 @@
 
 from collections import OrderedDict
 
-import cimodel.conf_tree as conf_tree
-import cimodel.miniutils as miniutils
-import cimodel.make_build_configs as make_build_configs
-import cimodel.visualization as visualization
+import cimodel.data.binary_build_data as binary_build_data
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
 
 
 class Conf(object):
@@ -19,7 +19,7 @@ class Conf(object):
         self.libtorch_variant = libtorch_variant
 
     def gen_build_env_parms(self):
-        return [self.pydistro] + self.parms + [make_build_configs.get_processor_arch_name(self.cuda_version)]
+        return [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
 
     def gen_docker_image(self):
 
@@ -50,40 +50,28 @@ class Conf(object):
 
     def gen_yaml_tree(self, build_or_test):
 
-        env_dict = OrderedDict({
-            "BUILD_ENVIRONMENT": miniutils.quote(" ".join(self.gen_build_env_parms())),
-        })
+        env_tuples = [("BUILD_ENVIRONMENT", miniutils.quote(" ".join(self.gen_build_env_parms())))]
 
         if self.libtorch_variant:
-            env_dict["LIBTORCH_VARIANT"] = miniutils.quote(self.libtorch_variant)
+            env_tuples.append(("LIBTORCH_VARIANT", miniutils.quote(self.libtorch_variant)))
 
-        os_word_substitution = {
-            "macos": "mac",
-        }
-
-        os_name = miniutils.override(self.os, os_word_substitution)
-
-        d = {
-            "environment": env_dict,
-            "<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test]),
-        }
+        os_name = miniutils.override(self.os, {"macos": "mac"})
+        d = {"<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test])}
 
         if build_or_test == "test":
-            tuples = []
 
             if not (self.smoke and self.os == "macos"):
-                tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+                env_tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
 
             if self.cuda_version:
-                tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
-
-            for (k, v) in tuples:
-                env_dict[k] = v
+                env_tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
 
         else:
             if self.os == "linux" and build_or_test != "upload":
                 d["docker"] = [{"image": self.gen_docker_image()}]
 
+        d["environment"] = OrderedDict(env_tuples)
+
         if build_or_test == "test":
             if self.cuda_version:
                 d["resource_class"] = "gpu.medium"
@@ -93,9 +81,9 @@ class Conf(object):
 
 def get_root(smoke, name):
 
-    return make_build_configs.TopLevelNode(
+    return binary_build_data.TopLevelNode(
         name,
-        make_build_configs.CONFIG_TREE_DATA,
+        binary_build_data.CONFIG_TREE_DATA,
         smoke,
     )
 
diff --git a/.circleci/cimodel/caffe2_build_definitions.py b/.circleci/cimodel/data/caffe2_build_definitions.py
index 6ee76b0c77..30647c542a 100644
--- a/.circleci/cimodel/caffe2_build_definitions.py
+++ b/.circleci/cimodel/data/caffe2_build_definitions.py
@@ -2,9 +2,9 @@
 
 from collections import OrderedDict
 
-import cimodel.dimensions as dimensions
-import cimodel.miniutils as miniutils
-from cimodel.conf_tree import Ver
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.miniutils as miniutils
+from cimodel.lib.conf_tree import Ver
 
 
 DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
@@ -126,33 +126,22 @@ class Conf(object):
             tuples.append(("BUILD_IOS", miniutils.quote("1")))
 
         if self.phase == "test":
-            use_cuda_docker = self.compiler.name == "cuda"
-            if use_cuda_docker:
+            # TODO cuda should not be considered a compiler
+            if self.compiler.name == "cuda":
                 tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
 
-        if not self.distro.name == "macos":
-            tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
-
-        if self.is_build_only():
-            if not self.distro.name == "macos":
-                tuples.append(("BUILD_ONLY", miniutils.quote("1")))
-
-        # TODO: not sure we need the distinction between system and homebrew anymore. Our python handling in cmake
-        #  and setuptools is more robust now than when we first had these.
         if self.distro.name == "macos":
-            tuples.append(("PYTHON_INSTALLATION", miniutils.quote("system")))
             tuples.append(("PYTHON_VERSION", miniutils.quote("2")))
 
-        env_dict = OrderedDict(tuples)
+        else:
+            tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+            if self.is_build_only():
+                tuples.append(("BUILD_ONLY", miniutils.quote("1")))
 
-        d = OrderedDict([
-            ("environment", env_dict),
-        ])
+        d = OrderedDict({"environment": OrderedDict(tuples)})
 
         if self.phase == "test":
-            is_large = self.compiler.name != "cuda"
-
-            resource_class = "large" if is_large else "gpu.medium"
+            resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
             d["resource_class"] = resource_class
 
         d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), self.phase, "defaults"])
diff --git a/.circleci/cimodel/data/dimensions.py b/.circleci/cimodel/data/dimensions.py
new file mode 100644
index 0000000000..edf297de49
--- /dev/null
+++ b/.circleci/cimodel/data/dimensions.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+
+PHASES = ["build", "test"]
+
+CUDA_VERSIONS = [
+    None,  # cpu build
+    "80",
+    "90",
+    "100",
+]
+
+STANDARD_PYTHON_VERSIONS = [
+    "2.7",
+    "3.5",
+    "3.6",
+    "3.7",
+]
diff --git a/.circleci/cimodel/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py
index 4e94b5b285..48bfb67e8f 100644
--- a/.circleci/cimodel/pytorch_build_definitions.py
+++ b/.circleci/cimodel/data/pytorch_build_definitions.py
@@ -2,11 +2,11 @@
 
 from collections import OrderedDict
 
-import cimodel.conf_tree as conf_tree
-import cimodel.dimensions as dimensions
-import cimodel.miniutils as miniutils
-import cimodel.visualization as visualization
-from cimodel.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
+from cimodel.lib.conf_tree import ConfigNode
 
 
 DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
@@ -129,11 +129,7 @@ class HiddenConf(object):
 
     def gen_workflow_yaml_item(self, phase):
 
-        val = OrderedDict()
-        dependency_build = self.parent_build
-        val["requires"] = [dependency_build.gen_build_name("build")]
-
-        return {self.gen_build_name(phase): val}
+        return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
 
     def gen_build_name(self, _):
         return self.name
@@ -153,10 +149,10 @@ def gen_dependent_configs(xenial_parent_config):
     for parms, gpu in extra_parms:
 
         c = Conf(
-            "xenial",
+            xenial_parent_config.distro,
             ["py3"] + parms,
             pyver="3.6",
-            cuda_version="8",
+            cuda_version=xenial_parent_config.cuda_version,
             restrict_phases=["test"],
             gpu_resource=gpu,
             parent_build=xenial_parent_config,
@@ -170,25 +166,33 @@ def gen_dependent_configs(xenial_parent_config):
     return configs
 
 
-# TODO make the schema consistent between "trusty" and "xenial"
+def X(val):
+    """
+    Compact way to write a leaf node
+    """
+    return val, []
+
+
 CONFIG_TREE_DATA = [
     ("trusty", [
-        ("2.7.9", []),
-        ("2.7", []),
-        ("3.5", []),
-        ("3.6", [
-            ("gcc4.8", []),
-            ("gcc5.4", [False, True]),
-            ("gcc7", []),
+        (None, [
+            X("2.7.9"),
+            X("2.7"),
+            X("3.5"),
+            X("nightly"),
+        ]),
+        ("gcc", [
+            ("4.8", [X("3.6")]),
+            ("5.4", [("3.6", [X(False), X(True)])]),
+            ("7", [X("3.6")]),
         ]),
-        ("nightly", []),
     ]),
     ("xenial", [
         ("clang", [
-            ("5", [("3.6", [])]),
+            ("5", [X("3.6")]),
         ]),
         ("cuda", [
-            ("8", [("3.6", [])]),
+            ("8", [X("3.6")]),
             ("9", [
                 # Note there are magic strings here
                 # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
@@ -197,11 +201,11 @@ CONFIG_TREE_DATA = [
                 # and
                 # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
                 # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
-                ("2.7", []),
-                ("3.6", []),
+                X("2.7"),
+                X("3.6"),
             ]),
-            ("9.2", [("3.6", [])]),
-            ("10", [("3.6", [])]),
+            ("9.2", [X("3.6")]),
+            ("10", [X("3.6")]),
         ]),
     ]),
 ]
@@ -222,99 +226,92 @@ def gen_tree():
     return configs_list
 
 
-class TopLevelNode(ConfigNode):
-    def __init__(self, node_name, config_tree_data):
-        super(TopLevelNode, self).__init__(None, node_name)
-
-        self.config_tree_data = config_tree_data
-
-    def get_children(self):
-        return [DistroConfigNode(self, d, p) for (d, p) in self.config_tree_data]
-
+class TreeConfigNode(ConfigNode):
+    def __init__(self, parent, node_name, subtree):
+        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+        self.subtree = subtree
+        self.init2(node_name)
 
-class DistroConfigNode(ConfigNode):
-    def __init__(self, parent, distro_name, subtree):
-        super(DistroConfigNode, self).__init__(parent, distro_name)
+    def modify_label(self, label):
+        return label
 
-        self.subtree = subtree
-        self.props["distro_name"] = distro_name
+    def init2(self, node_name):
+        pass
 
     def get_children(self):
+        return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
 
-        if self.find_prop("distro_name") == "trusty":
-            return [PyVerConfigNode(self, k, v) for k, v in self.subtree]
-        else:
-            return [XenialCompilerConfigNode(self, v, subtree) for (v, subtree) in self.subtree]
 
+class TopLevelNode(TreeConfigNode):
+    def __init__(self, node_name, subtree):
+        super(TopLevelNode, self).__init__(None, node_name, subtree)
 
-class PyVerConfigNode(ConfigNode):
-    def __init__(self, parent, pyver, subtree):
-        super(PyVerConfigNode, self).__init__(parent, pyver)
+    def child_constructor(self):
+        return DistroConfigNode
 
-        self.subtree = subtree
-        self.props["pyver"] = pyver
-
-        self.props["abbreviated_pyver"] = get_major_pyver(pyver)
 
-    def get_children(self):
-        return [CompilerConfigNode(self, v, xla_options) for (v, xla_options) in self.subtree]
+class DistroConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["distro_name"] = node_name
 
+    def child_constructor(self):
+        distro = self.find_prop("distro_name")
+        return TrustyCompilerConfigNode if distro == "trusty" else XenialCompilerConfigNode
 
-class CompilerConfigNode(ConfigNode):
-    def __init__(self, parent, compiler_name, subtree):
-        super(CompilerConfigNode, self).__init__(parent, compiler_name)
 
-        self.props["compiler_name"] = compiler_name
+class TrustyCompilerConfigNode(TreeConfigNode):
 
-        self.subtree = subtree
+    def modify_label(self, label):
+        return label or "<unspecified>"
 
-    def get_children(self):
-        return [XlaConfigNode(self, v) for v in self.subtree]
+    def init2(self, node_name):
+        self.props["compiler_name"] = node_name
 
+    def child_constructor(self):
+        return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
 
-class XenialCompilerConfigNode(ConfigNode):
-    def __init__(self, parent, compiler_name, subtree):
-        super(XenialCompilerConfigNode, self).__init__(parent, compiler_name)
 
-        self.props["compiler_name"] = compiler_name
+class TrustyCompilerVersionConfigNode(TreeConfigNode):
 
-        self.subtree = subtree
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
 
-    def get_children(self):
-        return [XenialCompilerVersionConfigNode(self, k, v) for (k, v) in self.subtree]
+    def child_constructor(self):
+        return PyVerConfigNode
 
 
-class XenialCompilerVersionConfigNode(ConfigNode):
-    def __init__(self, parent, compiler_version, subtree):
-        super(XenialCompilerVersionConfigNode, self).__init__(parent, compiler_version)
+class PyVerConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["pyver"] = node_name
+        self.props["abbreviated_pyver"] = get_major_pyver(node_name)
 
-        self.subtree = subtree
+    def child_constructor(self):
+        return XlaConfigNode
 
-        self.props["compiler_version"] = compiler_version
 
-    def get_children(self):
-        return [XenialPythonVersionConfigNode(self, v) for (v, _) in self.subtree]
+class XlaConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "XLA=" + str(label)
 
+    def init2(self, node_name):
+        self.props["is_xla"] = node_name
 
-class XenialPythonVersionConfigNode(ConfigNode):
-    def __init__(self, parent, python_version):
-        super(XenialPythonVersionConfigNode, self).__init__(parent, python_version)
 
-        self.props["pyver"] = python_version
-        self.props["abbreviated_pyver"] = get_major_pyver(python_version)
+class XenialCompilerConfigNode(TreeConfigNode):
 
-    def get_children(self):
-        return []
+    def init2(self, node_name):
+        self.props["compiler_name"] = node_name
 
+    def child_constructor(self):
+        return XenialCompilerVersionConfigNode
 
-class XlaConfigNode(ConfigNode):
-    def __init__(self, parent, xla_enabled):
-        super(XlaConfigNode, self).__init__(parent, "XLA=" + str(xla_enabled))
 
-        self.props["is_xla"] = xla_enabled
+class XenialCompilerVersionConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
 
-    def get_children(self):
-        return []
+    def child_constructor(self):
+        return PyVerConfigNode
 
 
 def instantiate_configs():
@@ -330,18 +327,17 @@ def instantiate_configs():
         python_version = None
         if distro_name == "xenial":
             python_version = fc.find_prop("pyver")
-
-        if distro_name == "xenial":
             parms_list = [fc.find_prop("abbreviated_pyver")]
         else:
             parms_list = ["py" + fc.find_prop("pyver")]
 
+        compiler_name = fc.find_prop("compiler_name")
+
         cuda_version = None
-        if fc.find_prop("compiler_name") == "cuda":
+        if compiler_name == "cuda":
             cuda_version = fc.find_prop("compiler_version")
 
-        compiler_name = fc.find_prop("compiler_name")
-        if compiler_name and compiler_name != "cuda":
+        elif compiler_name:
             gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
             parms_list.append(gcc_version)
 
@@ -381,7 +377,6 @@ def add_build_env_defs(jobs_dict):
     mydict = OrderedDict()
 
     config_list = instantiate_configs()
-
     for c in config_list:
 
         for phase in dimensions.PHASES:
@@ -427,7 +422,7 @@ def get_workflow_list():
             x.append(conf_options.gen_workflow_yaml_item(phase))
 
         # TODO convert to recursion
-        for conf in conf_options.dependent_tests:
+        for conf in conf_options.get_dependents():
             x.append(conf.gen_workflow_yaml_item("test"))
 
     return x
diff --git a/.circleci/cimodel/dimensions.py b/.circleci/cimodel/dimensions.py
deleted file mode 100644
index ae7840ca5d..0000000000
--- a/.circleci/cimodel/dimensions.py
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env python3
-
-
-PHASES = ["build", "test"]
diff --git a/.circleci/cimodel/lib/__init__.py b/.circleci/cimodel/lib/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/.circleci/cimodel/lib/__init__.py
diff --git a/.circleci/cimodel/conf_tree.py b/.circleci/cimodel/lib/conf_tree.py
index 9548ec775c..8a1caefd98 100644
--- a/.circleci/cimodel/conf_tree.py
+++ b/.circleci/cimodel/lib/conf_tree.py
@@ -22,6 +22,7 @@ class ConfigNode(object):
     def get_label(self):
         return self.node_name
 
+    # noinspection PyMethodMayBeStatic
     def get_children(self):
         return []
 
diff --git a/.circleci/cimodel/miniutils.py b/.circleci/cimodel/lib/miniutils.py
index dd21cd477b..b10fc448df 100644
--- a/.circleci/cimodel/miniutils.py
+++ b/.circleci/cimodel/lib/miniutils.py
@@ -2,7 +2,11 @@
 
 
 def quote(s):
-    return '"' + s + '"'
+    return sandwich('"', s)
+
+
+def sandwich(bread, jam):
+    return bread + jam + bread
 
 
 def override(word, substitutions):
diff --git a/.circleci/cimodel/miniyaml.py b/.circleci/cimodel/lib/miniyaml.py
index e4de65bf87..e4de65bf87 100644
--- a/.circleci/cimodel/miniyaml.py
+++ b/.circleci/cimodel/lib/miniyaml.py
diff --git a/.circleci/cimodel/visualization.py b/.circleci/cimodel/lib/visualization.py
index 2f472c2cf7..c842bf5fdd 100644
--- a/.circleci/cimodel/visualization.py
+++ b/.circleci/cimodel/lib/visualization.py
@@ -6,7 +6,7 @@ This module encapsulates dependencies on pygraphviz
 
 import colorsys
 
-import cimodel.conf_tree as conf_tree
+import cimodel.lib.conf_tree as conf_tree
 
 
 def rgb2hex(rgb_tuple):
@@ -16,6 +16,25 @@ def rgb2hex(rgb_tuple):
     return "#" + "".join(map(to_hex, list(rgb_tuple)))
 
 
+def handle_missing_graphviz(f):
+    """
+    If the user has not installed pygraphviz, this causes
+    calls to the draw() method of the returned object to do nothing.
+    """
+    try:
+        import pygraphviz
+        return f
+
+    except ModuleNotFoundError:
+
+        class FakeGraph:
+            def draw(self, *args, **kwargs):
+                pass
+
+        return lambda _: FakeGraph()
+
+
+@handle_missing_graphviz
 def generate_graph(toplevel_config_node):
     """
     Traverses the graph once first just to find the max depth
@@ -27,9 +46,9 @@ def generate_graph(toplevel_config_node):
     for config in config_list:
         max_depth = max(max_depth, config.get_depth())
 
-    from pygraphviz import AGraph
-
     # color the nodes using the max depth
+
+    from pygraphviz import AGraph
     dot = AGraph()
 
     def node_discovery_callback(node, sibling_index, sibling_count):
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 748ed4eb45..f9c23b4db2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,8 +5,8 @@
 # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
 # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
 # and then update DOCKER_IMAGE_VERSION at the top of the following files:
-# * cimodel/pytorch_build_definitions.py
-# * cimodel/caffe2_build_definitions.py
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
 
 docker_config_defaults: &docker_config_defaults
   user: jenkins
@@ -1103,10 +1103,8 @@ smoke_mac_test: &smoke_mac_test
 
 
 ##############################################################################
-##############################################################################
 # Job specifications job specs
 ##############################################################################
-##############################################################################
 version: 2
 jobs:
   pytorch_linux_trusty_py2_7_9_build:
@@ -1148,6 +1146,19 @@ jobs:
     resource_class: large
     <<: *pytorch_linux_test_defaults
 
+  pytorch_linux_trusty_pynightly_build:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_trusty_pynightly_test:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
+    resource_class: large
+    <<: *pytorch_linux_test_defaults
+
   pytorch_linux_trusty_py3_6_gcc4_8_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-trusty-py3.6-gcc4.8-build
@@ -1200,19 +1211,6 @@ jobs:
     resource_class: large
     <<: *pytorch_linux_test_defaults
 
-  pytorch_linux_trusty_pynightly_build:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
-    <<: *pytorch_linux_build_defaults
-
-  pytorch_linux_trusty_pynightly_test:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
-    resource_class: large
-    <<: *pytorch_linux_test_defaults
-
   pytorch_linux_xenial_py3_clang5_asan_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-asan-build
@@ -1677,14 +1675,12 @@ jobs:
     environment:
       BUILD_ENVIRONMENT: caffe2-py2-ios-macos10.13-build
       BUILD_IOS: "1"
-      PYTHON_INSTALLATION: "system"
       PYTHON_VERSION: "2"
     <<: *caffe2_macos_build_defaults
 
   caffe2_py2_system_macos10_13_build:
     environment:
       BUILD_ENVIRONMENT: caffe2-py2-system-macos10.13-build
-      PYTHON_INSTALLATION: "system"
       PYTHON_VERSION: "2"
     <<: *caffe2_macos_build_defaults
 
@@ -2040,10 +2036,11 @@ jobs:
       BUILD_ENVIRONMENT: "libtorch 2.7 cpu"
     <<: *binary_mac_build
 
-  # Binary build tests
-  # These are the smoke tests run right after the build, before the upload. If
-  # these fail, the upload doesn't happen
-  #############################################################################
+##############################################################################
+# Binary build tests
+# These are the smoke tests run right after the build, before the upload.
+# If these fail, the upload doesn't happen.
+##############################################################################
   binary_linux_manywheel_2.7m_cpu_test:
     environment:
       BUILD_ENVIRONMENT: "manywheel 2.7m cpu"
@@ -2339,8 +2336,9 @@ jobs:
 #    resource_class: gpu.medium
 #    <<: *binary_linux_test
 
-  # Binary build uploads
-  #############################################################################
+##############################################################################
+# Binary build uploads
+##############################################################################
   binary_linux_manywheel_2.7m_cpu_upload:
     environment:
       BUILD_ENVIRONMENT: "manywheel 2.7m cpu"
@@ -2586,7 +2584,6 @@ jobs:
       BUILD_ENVIRONMENT: "libtorch 2.7 cpu"
     <<: *binary_mac_upload
 
-
 ##############################################################################
 # Smoke test specs individual job specifications
 ##############################################################################
@@ -3065,6 +3062,10 @@ workflows:
       - pytorch_linux_trusty_py3_5_test:
           requires:
             - pytorch_linux_trusty_py3_5_build
+      - pytorch_linux_trusty_pynightly_build
+      - pytorch_linux_trusty_pynightly_test:
+          requires:
+            - pytorch_linux_trusty_pynightly_build
       - pytorch_linux_trusty_py3_6_gcc4_8_build
       - pytorch_linux_trusty_py3_6_gcc4_8_test:
           requires:
@@ -3081,10 +3082,6 @@ workflows:
       - pytorch_linux_trusty_py3_6_gcc7_test:
           requires:
             - pytorch_linux_trusty_py3_6_gcc7_build
-      - pytorch_linux_trusty_pynightly_build
-      - pytorch_linux_trusty_pynightly_test:
-          requires:
-            - pytorch_linux_trusty_pynightly_build
       - pytorch_linux_xenial_py3_clang5_asan_build
       - pytorch_linux_xenial_py3_clang5_asan_test:
           requires:
@@ -3270,7 +3267,6 @@ workflows:
       - smoke_macos_conda_3.7_cpu
       - smoke_macos_libtorch_2.7_cpu
 
-
 ##############################################################################
 # Daily binary build trigger
 ##############################################################################
@@ -3333,7 +3329,9 @@ workflows:
       - binary_macos_conda_3.7_cpu_build
       - binary_macos_libtorch_2.7_cpu_build
 
-      # Nightly tests
+##############################################################################
+# Nightly tests
+##############################################################################
       - binary_linux_manywheel_2.7m_cpu_test:
           requires:
             - binary_linux_manywheel_2.7m_cpu_build
diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py
index 06bce6f75b..f239c3654f 100755
--- a/.circleci/generate_config_yml.py
+++ b/.circleci/generate_config_yml.py
@@ -3,18 +3,18 @@
 """
 This script is the source of truth for config.yml.
 Please see README.md in this directory for details.
-
-In this module,
 """
 
 import os
 import sys
-from collections import OrderedDict
+import shutil
+from collections import namedtuple, OrderedDict
 
-import cimodel.pytorch_build_definitions as pytorch_build_definitions
-import cimodel.binary_build_definitions as binary_build_definitions
-import cimodel.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.miniyaml as miniyaml
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
+import cimodel.data.binary_build_definitions as binary_build_definitions
+import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.miniyaml as miniyaml
 
 
 class File(object):
@@ -26,16 +26,17 @@ class File(object):
 
     def write(self, output_filehandle):
         with open(os.path.join("verbatim-sources", self.filename)) as fh:
-            output_filehandle.write(fh.read())
+            shutil.copyfileobj(fh, output_filehandle)
+
+
+class FunctionGen(namedtuple('FunctionGen', 'function depth')):
+    __slots__ = ()
 
 
-class Treegen(object):
+class Treegen(FunctionGen):
     """
     Insert the content of a YAML tree into config.yml
     """
-    def __init__(self, function, depth):
-        self.function = function
-        self.depth = depth
 
     def write(self, output_filehandle):
         build_dict = OrderedDict()
@@ -43,18 +44,33 @@ class Treegen(object):
         miniyaml.render(output_filehandle, build_dict, self.depth)
 
 
-class Listgen(object):
+class Listgen(FunctionGen):
     """
     Insert the content of a YAML list into config.yml
     """
-    def __init__(self, function, depth):
-        self.function = function
-        self.depth = depth
-
     def write(self, output_filehandle):
         miniyaml.render(output_filehandle, self.function(), self.depth)
 
 
+def horizontal_rule():
+    return "".join("#" * 78)
+
+
+class Header(object):
+
+    def __init__(self, title, summary=None):
+        self.title = title
+        self.summary_lines = summary or []
+
+    def write(self, output_filehandle):
+        text_lines = [self.title] + self.summary_lines
+        comment_lines = ["# " + x for x in text_lines]
+        lines = miniutils.sandwich([horizontal_rule()], comment_lines)
+
+        for line in filter(None, lines):
+            output_filehandle.write(line + "\n")
+
+
 # Order of this list matters to the generated config.yml.
 YAML_SOURCES = [
     File("header-section.yml"),
@@ -64,30 +80,35 @@ YAML_SOURCES = [
     File("linux-binary-build-defaults.yml"),
     File("macos-binary-build-defaults.yml"),
     File("nightly-build-smoke-tests-defaults.yml"),
-    File("job-specs-header.yml"),
+    Header("Job specifications job specs"),
     Treegen(pytorch_build_definitions.add_build_env_defs, 0),
     File("job-specs-custom.yml"),
     Treegen(caffe2_build_definitions.add_caffe2_builds, 1),
     File("job-specs-html-update.yml"),
-    File("binary-build-specs-header.yml"),
+    Header("Binary build specs individual job specifications"),
     Treegen(binary_build_definitions.add_binary_build_specs, 1),
-    File("binary-build-tests-header.yml"),
+    Header(
+        "Binary build tests", [
+            "These are the smoke tests run right after the build, before the upload.",
+            "If these fail, the upload doesn't happen."
+        ]
+    ),
     Treegen(binary_build_definitions.add_binary_build_tests, 1),
     File("binary-build-tests.yml"),
-    File("binary-build-uploads-header.yml"),
+    Header("Binary build uploads"),
     Treegen(binary_build_definitions.add_binary_build_uploads, 1),
-    File("smoke-test-specs-header.yml"),
+    Header("Smoke test specs individual job specifications"),
     Treegen(binary_build_definitions.add_smoke_test_specs, 1),
     File("workflows.yml"),
     Listgen(pytorch_build_definitions.get_workflow_list, 3),
     File("workflows-pytorch-macos-builds.yml"),
     Listgen(caffe2_build_definitions.get_caffe2_workflows, 3),
     File("workflows-binary-builds-smoke-subset.yml"),
-    File("workflows-binary-smoke-header.yml"),
+    Header("Daily smoke test trigger"),
     Treegen(binary_build_definitions.add_binary_smoke_test_jobs, 1),
-    File("workflows-binary-build-header.yml"),
+    Header("Daily binary build trigger"),
     Treegen(binary_build_definitions.add_binary_build_jobs, 1),
-    File("workflows-nightly-tests-header.yml"),
+    Header("Nightly tests"),
     Listgen(binary_build_definitions.get_nightly_tests, 3),
     File("workflows-nightly-uploads-header.yml"),
     Listgen(binary_build_definitions.get_nightly_uploads, 3),
diff --git a/.circleci/regenerate.sh b/.circleci/regenerate.sh
index 5afaadff7a..43eff74748 100755
--- a/.circleci/regenerate.sh
+++ b/.circleci/regenerate.sh
@@ -3,4 +3,6 @@
 # Allows this script to be invoked from any directory:
 cd $(dirname "$0")
 
-./generate_config_yml.py > config.yml
+NEW_FILE=$(mktemp)
+./generate_config_yml.py > $NEW_FILE
+cp $NEW_FILE config.yml
diff --git a/.circleci/verbatim-sources/binary-build-specs-header.yml b/.circleci/verbatim-sources/binary-build-specs-header.yml
deleted file mode 100644
index d6cbd36fe3..0000000000
--- a/.circleci/verbatim-sources/binary-build-specs-header.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-##############################################################################
-# Binary build specs individual job specifications
-##############################################################################
diff --git a/.circleci/verbatim-sources/binary-build-tests-header.yml b/.circleci/verbatim-sources/binary-build-tests-header.yml
deleted file mode 100644
index 379f3b533f..0000000000
--- a/.circleci/verbatim-sources/binary-build-tests-header.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-  # Binary build tests
-  # These are the smoke tests run right after the build, before the upload. If
-  # these fail, the upload doesn't happen
-  #############################################################################
diff --git a/.circleci/verbatim-sources/binary-build-uploads-header.yml b/.circleci/verbatim-sources/binary-build-uploads-header.yml
deleted file mode 100644
index d316695729..0000000000
--- a/.circleci/verbatim-sources/binary-build-uploads-header.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-  # Binary build uploads
-  #############################################################################
diff --git a/.circleci/verbatim-sources/header-section.yml b/.circleci/verbatim-sources/header-section.yml
index a178109a46..6febfb9c55 100644
--- a/.circleci/verbatim-sources/header-section.yml
+++ b/.circleci/verbatim-sources/header-section.yml
@@ -5,8 +5,8 @@
 # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
 # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
 # and then update DOCKER_IMAGE_VERSION at the top of the following files:
-# * cimodel/pytorch_build_definitions.py
-# * cimodel/caffe2_build_definitions.py
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
 
 docker_config_defaults: &docker_config_defaults
   user: jenkins
diff --git a/.circleci/verbatim-sources/job-specs-header.yml b/.circleci/verbatim-sources/job-specs-header.yml
deleted file mode 100644
index 2d5618f6d9..0000000000
--- a/.circleci/verbatim-sources/job-specs-header.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-##############################################################################
-##############################################################################
-# Job specifications job specs
-##############################################################################
-##############################################################################
diff --git a/.circleci/verbatim-sources/smoke-test-specs-header.yml b/.circleci/verbatim-sources/smoke-test-specs-header.yml
deleted file mode 100644
index 8817d33c11..0000000000
--- a/.circleci/verbatim-sources/smoke-test-specs-header.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-##############################################################################
-# Smoke test specs individual job specifications
-##############################################################################
diff --git a/.circleci/verbatim-sources/workflows-binary-smoke-header.yml b/.circleci/verbatim-sources/workflows-binary-smoke-header.yml
deleted file mode 100644
index a3c7752be0..0000000000
--- a/.circleci/verbatim-sources/workflows-binary-smoke-header.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-##############################################################################
-# Daily smoke test trigger
-##############################################################################
diff --git a/.circleci/verbatim-sources/workflows-nightly-tests-header.yml b/.circleci/verbatim-sources/workflows-nightly-tests-header.yml
deleted file mode 100644
index 285493f783..0000000000
--- a/.circleci/verbatim-sources/workflows-nightly-tests-header.yml
+++ /dev/null
@@ -1 +0,0 @@
-      # Nightly tests