Setup cipd bin before calling ninja/autoninja

Tested in led builds on chromium/tools/build repo.

# Linux
```
> led get-build 8811504372914233697 | led edit-recipe-bundle -O depot_tools="$HOME/depot_tools" | led launch
...
https://ci.chromium.org/swarming/task/5b749884a3f3fa11
```

# Windows
```
> led get-build 8811491780369249585 | led edit-recipe-bundle -O https://ci.chromium.org/swarming/task/5b74bc67963d7610
```

Bug: 931218
Change-Id: I4a967f7544f71a9fef8a44baad45196365659596

Recipe-Nontrivial-Roll: build
Recipe-Nontrivial-Roll: build_limited
Recipe-Nontrivial-Roll: chrome_release
Recipe-Nontrivial-Roll: infra
Change-Id: I4a967f7544f71a9fef8a44baad45196365659596
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3697958
Auto-Submit: Junji Watanabe <jwata@google.com>
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Commit-Queue: Josip Sokcevic <sokcevic@google.com>
Reviewed-by: Josip Sokcevic <sokcevic@google.com>

.gitattributes

@@ -29,10 +29,8 @@
 /roll-dep*                    recipes
 /yapf*                        recipes
 
-# Additional files for cipd_bin_setup.{sh, bat},
-# including cipd/cipd.bat, cipd_client_version* and cipd_manifest*.
-/cipd*          recipes
-/.cipd_impl.ps1 recipes
+# Ensure depot_tools/.cipd_bin in a recipe step.
+/cipd_manifest* recipes
 
 # Symlinks
 /cbuildbot      recipes

recipes/README.recipes.md

@@ -119,7 +119,7 @@ Sets a fixed revision for a single dependency using project revision
 properties.
 ### *recipe_modules* / [depot\_tools](/recipes/recipe_modules/depot_tools)
 
-[DEPS](/recipes/recipe_modules/depot_tools/__init__.py#7): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime]
+[DEPS](/recipes/recipe_modules/depot_tools/__init__.py#7): [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime]
 
 PYTHON_VERSION_COMPATIBILITY: PY2+3
 
@@ -128,19 +128,19 @@ the depot_tools repo.
 
 #### **class [DepotToolsApi](/recipes/recipe_modules/depot_tools/api.py#12)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
 
-&emsp; **@property**<br>&mdash; **def [autoninja\_path](/recipes/recipe_modules/depot_tools/api.py#45)(self):**
+&emsp; **@property**<br>&mdash; **def [autoninja\_path](/recipes/recipe_modules/depot_tools/api.py#50)(self):**
 
-&emsp; **@property**<br>&mdash; **def [cros\_path](/recipes/recipe_modules/depot_tools/api.py#26)(self):**
+&emsp; **@property**<br>&mdash; **def [cros\_path](/recipes/recipe_modules/depot_tools/api.py#30)(self):**
 
-&emsp; **@property**<br>&mdash; **def [download\_from\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#13)(self):**
+&emsp; **@property**<br>&mdash; **def [download\_from\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#17)(self):**
 
-&emsp; **@property**<br>&mdash; **def [gn\_py\_path](/recipes/recipe_modules/depot_tools/api.py#30)(self):**
+&emsp; **@property**<br>&mdash; **def [gn\_py\_path](/recipes/recipe_modules/depot_tools/api.py#34)(self):**
 
-&emsp; **@property**<br>&mdash; **def [gsutil\_py\_path](/recipes/recipe_modules/depot_tools/api.py#36)(self):**
+&emsp; **@property**<br>&mdash; **def [gsutil\_py\_path](/recipes/recipe_modules/depot_tools/api.py#40)(self):**
 
-&emsp; **@property**<br>&mdash; **def [ninja\_path](/recipes/recipe_modules/depot_tools/api.py#40)(self):**
+&emsp; **@property**<br>&mdash; **def [ninja\_path](/recipes/recipe_modules/depot_tools/api.py#44)(self):**
 
-&emsp; **@contextlib.contextmanager**<br>&mdash; **def [on\_path](/recipes/recipe_modules/depot_tools/api.py#54)(self):**
+&emsp; **@contextlib.contextmanager**<br>&mdash; **def [on\_path](/recipes/recipe_modules/depot_tools/api.py#60)(self):**
 
 Use this context manager to put depot_tools on $PATH.
 
@@ -151,13 +151,13 @@ with api.depot_tools.on_path():
   # run some steps
 ```
 
-&emsp; **@property**<br>&mdash; **def [presubmit\_support\_py\_path](/recipes/recipe_modules/depot_tools/api.py#50)(self):**
+&emsp; **@property**<br>&mdash; **def [presubmit\_support\_py\_path](/recipes/recipe_modules/depot_tools/api.py#56)(self):**
 
-&emsp; **@property**<br>&mdash; **def [root](/recipes/recipe_modules/depot_tools/api.py#21)(self):**
+&emsp; **@property**<br>&mdash; **def [root](/recipes/recipe_modules/depot_tools/api.py#25)(self):**
 
 Returns (Path): The "depot_tools" root directory.
 
-&emsp; **@property**<br>&mdash; **def [upload\_to\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#17)(self):**
+&emsp; **@property**<br>&mdash; **def [upload\_to\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#21)(self):**
 ### *recipe_modules* / [gclient](/recipes/recipe_modules/gclient)
 
 [DEPS](/recipes/recipe_modules/gclient/__init__.py#3): [git](#recipe_modules-git), [gitiles](#recipe_modules-gitiles), [tryserver](#recipe_modules-tryserver), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step]

recipes/recipe_modules/depot_tools/__init__.py

@@ -5,6 +5,7 @@
 PYTHON_VERSION_COMPATIBILITY = 'PY2+3'
 
 DEPS = [
+  'recipe_engine/cipd',
   'recipe_engine/context',
   'recipe_engine/platform',
   'recipe_engine/runtime',

recipes/recipe_modules/depot_tools/api.py

@@ -10,6 +10,10 @@ import contextlib
 from recipe_engine import recipe_api
 
 class DepotToolsApi(recipe_api.RecipeApi):
+  def __init__(self, **kwargs):
+    super(DepotToolsApi, self).__init__(**kwargs);
+    self._cipd_bin_setup_called = False
+
   @property
   def download_from_google_storage_path(self):
     return self.repo_resource('download_from_google_storage.py')
@@ -39,11 +43,13 @@ class DepotToolsApi(recipe_api.RecipeApi):
 
   @property
   def ninja_path(self):
+    self._cipd_bin_setup()
     ninja_exe = 'ninja.exe' if self.m.platform.is_win else 'ninja'
     return self.repo_resource(ninja_exe)
 
   @property
   def autoninja_path(self):
+    self._cipd_bin_setup()
     autoninja = 'autoninja.bat' if self.m.platform.is_win else 'autoninja'
     return self.repo_resource(autoninja)
 
@@ -70,3 +76,13 @@ class DepotToolsApi(recipe_api.RecipeApi):
             'DEPOT_TOOLS_UPDATE': '0'
         }}):
       yield
+
+  def _cipd_bin_setup(self):
+    """Installs CIPD packages under .cipd_bin."""
+    if self._cipd_bin_setup_called:
+      return
+    self.m.cipd.ensure(
+      self.repo_resource('.cipd_bin'),
+      self.repo_resource('cipd_manifest.txt'),
+      'ensure depot_tools/.cipd_bin')
+    self._cipd_bin_setup_called = True

recipes/recipe_modules/depot_tools/examples/full.expected/basic.json

@@ -41,6 +41,27 @@
     ],
     "name": "gsutil_py_path"
   },
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "RECIPE_REPO[depot_tools]/.cipd_bin",
+      "-ensure-file",
+      "RECIPE_REPO[depot_tools]/cipd_manifest.txt",
+      "-max-threads",
+      "0",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "ensure depot_tools/.cipd_bin",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {}@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
   {
     "cmd": [
       "ls",

recipes/recipe_modules/depot_tools/examples/full.expected/win.json

@@ -41,6 +41,27 @@
     ],
     "name": "gsutil_py_path"
   },
+  {
+    "cmd": [
+      "cipd.bat",
+      "ensure",
+      "-root",
+      "RECIPE_REPO[depot_tools]\\.cipd_bin",
+      "-ensure-file",
+      "RECIPE_REPO[depot_tools]\\cipd_manifest.txt",
+      "-max-threads",
+      "0",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "ensure depot_tools/.cipd_bin",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {}@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
   {
     "cmd": [
       "ls",