Merge pull request MariaDB#19 from mozilla/try-extender

Support add new jobs in Treeherder

Author: adusca

Procfile

@@ -1,3 +1,3 @@
 worker1: python pulse_actions/worker.py --topic-base backfilling --dry-run
-worker2: python pulse_actions/worker.py --topic-base resultset_actions,manual_backfill
+worker2: python pulse_actions/worker.py --topic-base resultset_actions,manual_backfill,runnable
 

pulse_actions/handlers/config.py

@@ -1,5 +1,6 @@
 import pulse_actions.handlers.treeherder_buildbot as treeherder_buildbot
 import pulse_actions.handlers.treeherder_resultset as treeherder_resultset
+import pulse_actions.handlers.treeherder_runnable as treeherder_runnable
 import pulse_actions.handlers.backfilling as backfilling
 
 
@@ -16,6 +17,12 @@
     "exchange/treeherder-stage/v1/resultset-actions": {
         "resultset_actions-stage": treeherder_resultset.on_resultset_action_stage_event
     },
+    "exchange/treeherder/v1/resultset-runnable-job-actions": {
+        "runnable": treeherder_runnable.on_runnable_job_prod_event
+    },
+    "exchange/treeherder-stage/v1/resultset-runnable-job-actions": {
+        "runnable-stage": treeherder_runnable.on_runnable_job_stage_event
+    },
     "exchange/build/normalized": {
         "backfilling": backfilling.on_event
     }

pulse_actions/handlers/route_functions.py

@@ -1,5 +1,8 @@
 import pulse_actions.handlers.treeherder_buildbot as treeherder_buildbot
 import pulse_actions.handlers.treeherder_resultset as treeherder_resultset
+import pulse_actions.handlers.treeherder_runnable as treeherder_runnable
+
+
 import logging
 
 LOG = logging.getLogger()
@@ -8,6 +11,8 @@
 def route(data, message, dry_run):
     if 'job_id' in data:
         treeherder_buildbot.on_buildbot_event(data, message, dry_run)
+    elif 'buildernames' in data:
+        treeherder_runnable.on_runnable_job_prod_event(data, message, dry_run)
     elif 'resultset_id' in data:
         treeherder_resultset.on_resultset_action_event(data, message, dry_run)
     else:

pulse_actions/handlers/treeherder_runnable.py

@@ -0,0 +1,80 @@
+import logging
+
+from mozci.ci_manager import TaskClusterBuildbotManager
+from mozci.sources import buildjson, buildbot_bridge
+from mozci import query_jobs
+from thclient import TreeherderClient
+from pulse_actions.publisher import MessageHandler
+
+logging.basicConfig(format='%(levelname)s:\t %(message)s')
+LOG = logging.getLogger()
+MEMORY_SAVING_MODE = True
+
+
+def on_runnable_job_stage_event(data, message, dry_run):
+    return on_runnable_job_event(data, message, dry_run, stage=True)
+
+
+def on_runnable_job_prod_event(data, message, dry_run):
+    return on_runnable_job_event(data, message, dry_run, stage=False)
+
+
+def on_runnable_job_event(data, message, dry_run, stage):
+    # Cleaning mozci caches
+    buildjson.BUILDS_CACHE = {}
+    query_jobs.JOBS_CACHE = {}
+    repo_name = data["project"]
+    requester = data["requester"]
+    resultset_id = data["resultset_id"]
+    buildernames = data["buildernames"]
+
+    if stage:
+        treeherder_client = TreeherderClient(host='treeherder.allizom.org')
+    else:
+        treeherder_client = TreeherderClient()
+
+    mgr = TaskClusterBuildbotManager()
+
+    LOG.info("New jobs requested by %s on repo_name %s with resultset_id: %s" %
+             (data["requester"], data["project"], data["resultset_id"]))
+    resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]
+    revision = resultset["revision"]
+    author = resultset["author"]
+    status = None
+
+    message_sender = MessageHandler()
+    # Everyone can press the button, but only authorized users can trigger jobs
+    # TODO: remove this when proper LDAP identication is set up on TH
+    if author != requester and not requester.endswith('@mozilla.com'):
+        message.ack()
+
+        # We publish a message saying we will not trigger the job
+        pulse_message = {
+            'resultset_id': resultset_id,
+            'requester': requester,
+            'status': "Could not determine if the user is authorized, nothing was triggered."}
+        routing_key = '{}.{}'.format(repo_name, 'runnable')
+        message_sender.publish_message(pulse_message, routing_key)
+
+        raise Exception("Requester %s is not allowed to trigger jobs." %
+                        requester)
+
+    builders_graph = buildbot_bridge.buildbot_graph_builder(buildernames, revision)
+    mgr.schedule_graph(
+        repo_name=repo_name,
+        revision=revision,
+        builders_graph=builders_graph,
+        dry_run=dry_run)
+
+    # Send a pulse message showing what we did
+    message_sender = MessageHandler()
+    pulse_message = {
+        'resultset_id': resultset_id,
+        'graph': builders_graph,
+        'requester': requester,
+        'status': status}
+    routing_key = '{}.{}'.format(repo_name, 'runnable')
+    message_sender.publish_message(pulse_message, routing_key)
+
+    # We need to ack the message to remove it from our queue
+    message.ack()

pulse_actions/run_time_config.json

@@ -15,8 +15,17 @@
         "exchange": "exchange/treeherder-stage/v1/resultset-actions",
         "topic": "#.#"
     },
-    "backfilling":{
+    "backfilling": {
         "exchange": "exchange/build/normalized",
         "topic": "unittest.mozilla-inbound.#"
+    },
+    "runnable": {
+        "exchange": "exchange/treeherder/v1/resultset-runnable-job-actions",
+        "topic": "#"
+    },
+    "runnable-stage": {
+        "exchange": "exchange/treeherder-stage/v1/resultset-runnable-job-actions",
+        "topic": "#"
     }
+
 }

requirements.txt

@@ -6,7 +6,7 @@ httplib2==0.9.2
 ijson==2.2
 keyring==5.4
 kombu==3.0.26
-mozci==0.18.0
+mozci==0.21.0
 MozillaPulse==1.2.2
 oauth2==1.9.0.post1
 progressbar==2.3