ninjalog uploader: migrate to python3

This removes dependency to httplib2 by using urllib.
https://docs.python.org/3/library/http.client.html#httpresponse-objects

Use six for both py2/py3 unittest.

Bug: b/177465438
Change-Id: I48383559842615f97abea45b55ca0acf4d7c8bd3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2629087
Commit-Queue: Bruce Dawson <brucedawson@chromium.org>
Reviewed-by: Bruce Dawson <brucedawson@chromium.org>
Reviewed-by: Yoshisato Yanagisawa <yyanagisawa@google.com>
Reviewed-by: Fumitoshi Ukai <ukai@google.com>
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>

autoninja

@@ -24,12 +24,12 @@ if eval "$command"; then
   fi
 
   # Collect ninjalog from googler.
-  python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
+  python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
   exit
 fi
 
 # Collect ninjalog from googler.
-python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
+python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
 
 # Return an error code of 1 so that if a developer types:
 # "autoninja chrome && chrome" then chrome won't run if the build fails.

autoninja.bat

@@ -36,14 +36,14 @@ REM Also print it to reassure that the right settings are being used.
 FOR /f "usebackq tokens=*" %%a in (`vpython %scriptdir%autoninja.py "%*"`) do echo %%a & %%a
 @if errorlevel 1 goto buildfailure
 
-REM Use call to invoke vpython script here, because we use vpython via vpython.bat.
+REM Use call to invoke python script here, because we may use python3 via python3.bat.
 @if "%NINJA_SUMMARIZE_BUILD%" == "1" call python3 %scriptdir%post_build_ninja_summary.py %*
-@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
+@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
 
 exit /b
 :buildfailure
 
-@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
+@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
 
 REM Return an error code of 1 so that if a developer types:
 REM "autoninja chrome && chrome" then chrome won't run if the build fails.

ninjalog_uploader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -15,8 +15,8 @@ The log will be used to analyze user side build performance.
 """
 
 import argparse
-import cStringIO
 import gzip
+import io
 import json
 import logging
 import multiprocessing
@@ -26,7 +26,8 @@ import subprocess
 import sys
 import time
 
-from third_party import httplib2
+from third_party.six.moves.urllib import error
+from third_party.six.moves.urllib import request
 
 # These build configs affect build performance a lot.
 # TODO(https://crbug.com/900161): Add 'blink_symbol_level' and
@@ -39,10 +40,9 @@ WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
 def IsGoogler(server):
   """Check whether this script run inside corp network."""
   try:
-    h = httplib2.Http()
-    _, content = h.request('https://' + server + '/should-upload', 'GET')
-    return content == 'Success'
-  except httplib2.HttpLib2Error:
+    resp = request.urlopen('https://' + server + '/should-upload')
+    return resp.read() == b'Success'
+  except error.URLError:
     return False
 
 
@@ -202,31 +202,28 @@ def main():
     logging.info("ninjalog is not updated recently %s", ninjalog)
     return 0
 
-  output = cStringIO.StringIO()
+  output = io.BytesIO()
 
   with open(ninjalog) as f:
     with gzip.GzipFile(fileobj=output, mode='wb') as g:
-      g.write(f.read())
-      g.write('# end of ninja log\n')
+      g.write(f.read().encode())
+      g.write(b'# end of ninja log\n')
 
       metadata = GetMetadata(args.cmdline, ninjalog)
       logging.info('send metadata: %s', json.dumps(metadata))
-      g.write(json.dumps(metadata))
-
-  h = httplib2.Http()
-  resp_headers, content = h.request('https://' + args.server +
-                                    '/upload_ninja_log/',
-                                    'POST',
-                                    body=output.getvalue(),
-                                    headers={'Content-Encoding': 'gzip'})
-
-  if resp_headers.status != 200:
-    logging.warning("unexpected status code for response: %s",
-                    resp_headers.status)
+      g.write(json.dumps(metadata).encode())
+
+  resp = request.urlopen(
+      request.Request('https://' + args.server + '/upload_ninja_log/',
+                      data=output.getvalue(),
+                      headers={'Content-Encoding': 'gzip'}))
+
+  if resp.status != 200:
+    logging.warning("unexpected status code for response: %s", resp.status)
     return 1
 
-  logging.info('response header: %s', resp_headers)
-  logging.info('response content: %s', content)
+  logging.info('response header: %s', resp.headers)
+  logging.info('response content: %s', resp.read())
   return 0
 
 

ninjalog_uploader_wrapper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -115,7 +115,7 @@ def main():
   creationnflags = 0
   if platform.system() == 'Windows':
     creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP
-  subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
+  subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:],
                     stdout=devnull,
                     stderr=devnull,
                     creationflags=creationnflags)

tests/ninjalog_uploader_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -15,105 +15,105 @@ import ninjalog_uploader
 
 
 class NinjalogUploaderTest(unittest.TestCase):
-    def test_parse_gn_args(self):
-        self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {})
-
-        # Extract current configs from GN's output json.
-        self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
-            {
-                'current': {'value': 'true'},
-                'default': {'value': 'false'},
-                'name': 'is_component_build'
-            },
-            {
-                'default': {'value': '"x64"'},
-                'name': 'host_cpu'
-            },
-        ])), {
-            'is_component_build': 'true',
-            'host_cpu': '"x64"',
-        })
-
-        self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
-            {
-                'current': {'value': 'true'},
-                'default': {'value': 'false'},
-                'name': 'is_component_build'
-            },
-            {
-                'current': {'value': 'false'},
-                'default': {'value': 'false'},
-                'name': 'use_goma'
-            },
-        ])), {'is_component_build': 'true',
-              'use_goma': 'false'})
-
-    def test_get_ninjalog(self):
-        # No args => default to cwd.
-        self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']),
-                         './.ninja_log')
-
-        # Specified by -C case.
-        self.assertEqual(
-            ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']),
-            'out/Release/.ninja_log')
-        self.assertEqual(
-            ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']),
-            'out/Release/.ninja_log')
-
-        # Invalid -C flag case.
-        self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']),
-                         './.ninja_log')
-
-        # Multiple target directories => use the last directory.
-        self.assertEqual(ninjalog_uploader.GetNinjalog(
-            ['ninja', '-C', 'out/Release', '-C', 'out/Debug']),
-            'out/Debug/.ninja_log')
-
-    def test_get_build_target_from_command_line(self):
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', 'chrome']), ['chrome'])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja']), [])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', '-j', '1000', 'chrome']), ['chrome'])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', 'chrome', '-j', '1000']), ['chrome'])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', '-C', 'chrome']), [])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', '-Cout/Release', 'chrome']), ['chrome'])
-
-        self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
-            ['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all'])
-
-    def test_get_j_flag(self):
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja']), None)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-j', '1000']), 1000)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-j', '1000a']), None)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-j', 'a']), None)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-j1000']), 1000)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-ja']), None)
-
-        self.assertEqual(ninjalog_uploader.GetJflag(
-            ['ninja','-j']), None)
+  def test_parse_gn_args(self):
+    self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {})
+
+    # Extract current configs from GN's output json.
+    self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
+        {
+            'current': {'value': 'true'},
+            'default': {'value': 'false'},
+            'name': 'is_component_build'
+        },
+        {
+            'default': {'value': '"x64"'},
+            'name': 'host_cpu'
+        },
+    ])), {
+        'is_component_build': 'true',
+        'host_cpu': '"x64"',
+    })
+
+    self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
+        {
+            'current': {'value': 'true'},
+            'default': {'value': 'false'},
+            'name': 'is_component_build'
+        },
+        {
+            'current': {'value': 'false'},
+            'default': {'value': 'false'},
+            'name': 'use_goma'
+        },
+    ])), {'is_component_build': 'true',
+          'use_goma': 'false'})
+
+  def test_get_ninjalog(self):
+    # No args => default to cwd.
+    self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']),
+                     './.ninja_log')
+
+    # Specified by -C case.
+    self.assertEqual(
+        ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']),
+        'out/Release/.ninja_log')
+    self.assertEqual(
+        ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']),
+        'out/Release/.ninja_log')
+
+    # Invalid -C flag case.
+    self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']),
+                     './.ninja_log')
+
+    # Multiple target directories => use the last directory.
+    self.assertEqual(ninjalog_uploader.GetNinjalog(
+        ['ninja', '-C', 'out/Release', '-C', 'out/Debug']),
+        'out/Debug/.ninja_log')
+
+  def test_get_build_target_from_command_line(self):
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', 'chrome']), ['chrome'])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja']), [])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', '-j', '1000', 'chrome']), ['chrome'])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', 'chrome', '-j', '1000']), ['chrome'])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', '-C', 'chrome']), [])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', '-Cout/Release', 'chrome']), ['chrome'])
+
+    self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
+        ['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all'])
+
+  def test_get_j_flag(self):
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja']), None)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-j', '1000']), 1000)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-j', '1000a']), None)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-j', 'a']), None)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-j1000']), 1000)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-ja']), None)
+
+    self.assertEqual(ninjalog_uploader.GetJflag(
+        ['ninja','-j']), None)
 
 
 if __name__ == '__main__':
-    unittest.main()
+  unittest.main()