Use json encoding to avoid spamming remote API (breaking)

checkbox-ng/checkbox_ng/launcher/controller.py

@@ -455,7 +455,8 @@
         configuration["normal_user"] = self._normal_user
         try:
             _logger.info("remote: Starting new session.")
-            tps = self.sa.start_session(configuration)
+            tps = self.sa.start_session_json(json.dumps(configuration))
+            tps = json.loads(tps)
             if self.sa.sideloaded_providers:
                 _logger.warning("Agent is using sideloaded providers")
         except RuntimeError as exc:
@@ -604,7 +605,7 @@
             _logger.error('The test plan "%s" is not available!', tp)
             raise SystemExit(1)
         self._is_bootstrapping = True
-        bs_todo = self.sa.get_bootstrapping_todo_list()
+        bs_todo = json.loads(self.sa.get_bootstrapping_todo_list_json())
         for job_no, job_id in enumerate(bs_todo, start=1):
             print(
                 self.C.header(
@@ -616,13 +617,14 @@
             self.sa.run_bootstrapping_job(job_id)
             self.wait_for_job()
         self._is_bootstrapping = False
-        self.jobs = self.sa.finish_bootstrap()
+        self.jobs = json.loads(self.sa.finish_bootstrap_json())
 
     def _strtobool(self, val):
         return val.lower() in ("y", "yes", "t", "true", "on", "1")
 
     def _save_manifest(self, interactive):
-        manifest_repr = self.sa.get_manifest_repr()
+        manifest_repr = self.sa.get_manifest_repr_json()
+        manifest_repr = json.loads(manifest_repr)
         if not manifest_repr:
             _logger.info("Skipping saving of the manifest")
             return
@@ -644,15 +646,17 @@
             to_save_manifest = {
                 conf["id"]: conf["value"] for conf in all_preconf
             }
-        self.sa.save_manifest(to_save_manifest)
+        self.sa.save_manifest_json(json.dumps(to_save_manifest))
 
     def select_jobs(self, all_jobs):
         if self.launcher.get_value("test selection", "forced"):
             if self.launcher.manifest:
                 self._save_manifest(interactive=False)
         else:
             _logger.info("controller: Selecting jobs.")
-            reprs = json.loads(self.sa.get_jobs_repr(all_jobs))
+            reprs = json.loads(
+                self.sa.get_jobs_repr_json(json.dumps(all_jobs))
+            )
             wanted_set = CategoryBrowser(
                 "Choose tests to run on your system:", reprs
             ).run()
@@ -662,7 +666,7 @@
                 # the original list
                 chosen_jobs = [job for job in all_jobs if job in wanted_set]
                 _logger.debug("controller: Selected jobs: %s", chosen_jobs)
-                self.sa.modify_todo_list(chosen_jobs)
+                self.sa.modify_todo_list_json(json.dumps(chosen_jobs))
             self._save_manifest(interactive=True)
         self.sa.finish_job_selection()
 
@@ -710,18 +714,7 @@
                 stack.callback(signal.signal, signal.SIGINT, tmp_sig)
                 self._export_results()
         # let's see if any of the jobs failed, if so, let's return an error code of 1
-        job_state_map = (
-            self._sa.manager.default_device_context._state._job_state_map
-        )
-        failing_outcomes = (
-            IJobResult.OUTCOME_FAIL,
-            IJobResult.OUTCOME_CRASH,
-        )
-        self._has_anything_failed = any(
-            job.result.outcome in failing_outcomes
-            for job in job_state_map.values()
-        )
-
+        self._has_anything_failed = self.sa.has_any_job_failed()
         self.sa.finalize_session()
         return False
 
@@ -760,15 +753,17 @@
         ):
             self._handle_last_job_after_resume(resumed_ongoing_session_info)
         _logger.info("controller: Running jobs.")
-        jobs = self.sa.get_session_progress()
+        jobs = json.loads(self.sa.get_session_progress_json())
         _logger.debug(
             "controller: Jobs to be run:\n%s",
             "\n".join(["  " + job for job in jobs]),
         )
         total_num = len(jobs["done"]) + len(jobs["todo"])
 
         jobs_repr = json.loads(
-            self.sa.get_jobs_repr(jobs["todo"], len(jobs["done"]))
+            self.sa.get_jobs_repr_json(
+                json.dumps(jobs["todo"]), len(jobs["done"])
+            )
         )
 
         self._run_jobs(jobs_repr, total_num)
@@ -790,9 +785,12 @@
 
     def wait_for_job(self, dont_finish=False):
         _logger.info("controller: Waiting for job to finish.")
+        polling_backoff = [0, 0.1, 0.2, 0.5]
+        polling_i = 0
         while True:
             state, payload = self.sa.monitor_job()
             if payload and not self._is_bootstrapping:
+                polling_i = 0
                 for line in payload.splitlines():
                     if line.startswith("stderr"):
                         SimpleUI.red_text(line[6:])
@@ -801,7 +799,8 @@
                     else:
                         SimpleUI.black_text(line[6:])
             if state == "running":
-                time.sleep(0.5)
+                time.sleep(polling_backoff[polling_i])
+                polling_i = min(polling_i + 1, len(polling_backoff) - 1)
                 while True:
                     res = select.select([sys.stdin], [], [], 0)
                     if not res[0]:
@@ -837,7 +836,7 @@
         _logger.info("controller: Exporting locally'")
         rf = self.sa.cache_report(exporter_id, options)
         exported_stream = SpooledTemporaryFile(max_size=102400, mode="w+b")
-        chunk_size = 16384
+        chunk_size = 160 * 1024
         with tqdm(
             total=rf.tell(),
             unit="B",
@@ -903,7 +902,8 @@
     def _run_jobs(self, jobs_repr, total_num=0):
         for job in jobs_repr:
             job_state = self.sa.get_job_state(job["id"])
-            self.sa.note_metadata_starting_job(job, job_state)
+            # Note: job_state is a remote object, no need to json encode it
+            self.sa.note_metadata_starting_job_json(json.dumps(job), job_state)
             SimpleUI.header(
                 _("Running job {} / {}").format(
                     job["num"], total_num, fill="-"
@@ -916,14 +916,13 @@
             next_job = False
             while next_job is False:
                 for interaction in self.sa.run_job(job["id"]):
-                    if interaction.kind == "purpose":
-                        SimpleUI.description(
-                            _("Purpose:"), interaction.message
-                        )
-                    elif interaction.kind == "description":
-                        SimpleUI.description(
-                            _("Description:"), interaction.message
-                        )
+                    # interaction is a netref, cache attributes here
+                    kind = interaction.kind
+                    message = interaction.message
+                    if kind == "purpose":
+                        SimpleUI.description(_("Purpose:"), message)
+                    elif kind == "description":
+                        SimpleUI.description(_("Description:"), message)
                         if job["command"] is None:
                             cmd = "run"
                         else:
@@ -937,8 +936,8 @@
                             raise SystemExit("Session paused by the user")
                         self.sa.remember_users_response(cmd)
                         self.wait_for_job(dont_finish=True)
-                    elif interaction.kind in "steps":
-                        SimpleUI.description(_("Steps:"), interaction.message)
+                    elif kind in "steps":
+                        SimpleUI.description(_("Steps:"), message)
                         if job["command"] is None:
                             cmd = "run"
                         else:
@@ -951,12 +950,10 @@
                             self.sa.remember_users_response(cmd)
                             raise SystemExit("Session paused by the user")
                         self.sa.remember_users_response(cmd)
-                    elif interaction.kind == "verification":
+                    elif kind == "verification":
                         self.wait_for_job(dont_finish=True)
-                        if interaction.message:
-                            SimpleUI.description(
-                                _("Verification:"), interaction.message
-                            )
+                        if message:
+                            SimpleUI.description(_("Verification:"), message)
                         JobAdapter = namedtuple("job_adapter", ["command"])
                         job_lite = JobAdapter(job["command"])
                         try:
@@ -976,14 +973,14 @@
                                 interaction.extra._builder.get_result(),
                             )
                             break
-                    elif interaction.kind == "comment":
+                    elif kind == "comment":
                         new_comment = input(
                             SimpleUI.C.BLUE(
                                 _("Please enter your comments:") + "\n"
                             )
                         )
                         self.sa.remember_users_response(new_comment + "\n")
-                    elif interaction.kind == "skip":
+                    elif kind == "skip":
                         if (
                             job_state.effective_certification_status
                             == "blocker"

checkbox-ng/checkbox_ng/launcher/stages.py

@@ -453,9 +453,18 @@
         self._export_fn = export_fn
 
     def _prepare_stock_report(self, report):
+        try:
+            # This function is called by both remote and local. Local sa
+            # doesn't have this API so it will fallback to the "normal"
+            # Configuration type. In remote we get the agent Configuration type
+            # because the object is passed and mostly used by the agent, so
+            # creating this object as a netref saves a lot of rpyc calls
+            ConfigurationType = self.sa.configuration_type()
+        except AttributeError:
+            ConfigurationType = Configuration
         new_origin = "stock_reports"
         if report == "text":
-            additional_config = Configuration.from_text(
+            additional_config = ConfigurationType.from_text(
                 textwrap.dedent(
                     """
                     [exporter:text]
@@ -473,7 +482,7 @@
             )
             self.sa.config.update_from_another(additional_config, new_origin)
         elif report == "certification":
-            additional_config = Configuration.from_text(
+            additional_config = ConfigurationType.from_text(
                 textwrap.dedent(
                     """
                     [exporter:tar]
@@ -489,7 +498,7 @@
             )
             self.sa.config.update_from_another(additional_config, new_origin)
         elif report == "certification-staging":
-            additional_config = Configuration.from_text(
+            additional_config = ConfigurationType.from_text(
                 textwrap.dedent(
                     """
                     [exporter:tar]
@@ -526,7 +535,7 @@
                     transport = {exporter}_file
                     """
                 )
-                additional_config = Configuration.from_text(
+                additional_config = ConfigurationType.from_text(
                     template.format(exporter=exporter, path=path), new_origin
                 )
                 self.sa.config.update_from_another(
@@ -550,7 +559,7 @@
                 transport = {exporter}_file
                 """
             )
-            additional_config = Configuration.from_text(
+            additional_config = ConfigurationType.from_text(
                 template.format(exporter=exporter, path=path), new_origin
             )
             self.sa.config.update_from_another(additional_config, new_origin)