gclient.py 58 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649
  1. #!/usr/bin/python
  2. #
  3. # Copyright 2008 Google Inc. All Rights Reserved.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. """A wrapper script to manage a set of client modules in different SCM.
  17. This script is intended to be used to help basic management of client
  18. program sources residing in one or more Subversion modules, along with
  19. other modules it depends on, also in Subversion, but possibly on
  20. multiple respositories, making a wrapper system apparently necessary.
  21. Files
  22. .gclient : Current client configuration, written by 'config' command.
  23. Format is a Python script defining 'solutions', a list whose
  24. entries each are maps binding the strings "name" and "url"
  25. to strings specifying the name and location of the client
  26. module, as well as "custom_deps" to a map similar to the DEPS
  27. file below.
  28. .gclient_entries : A cache constructed by 'update' command. Format is a
  29. Python script defining 'entries', a list of the names
  30. of all modules in the client
  31. <module>/DEPS : Python script defining var 'deps' as a map from each requisite
  32. submodule name to a URL where it can be found (via one SCM)
  33. Hooks
  34. .gclient and DEPS files may optionally contain a list named "hooks" to
  35. allow custom actions to be performed based on files that have changed in the
  36. working copy as a result of a "sync"/"update" or "revert" operation. Hooks
  37. can also be run based on what files have been modified in the working copy
  38. with the "runhooks" operation. If any of these operation are run with
  39. --force, all known hooks will run regardless of the state of the working
  40. copy.
  41. Each item in a "hooks" list is a dict, containing these two keys:
  42. "pattern" The associated value is a string containing a regular
  43. expression. When a file whose pathname matches the expression
  44. is checked out, updated, or reverted, the hook's "action" will
  45. run.
  46. "action" A list describing a command to run along with its arguments, if
  47. any. An action command will run at most one time per gclient
  48. invocation, regardless of how many files matched the pattern.
  49. The action is executed in the same directory as the .gclient
  50. file. If the first item in the list is the string "python",
  51. the current Python interpreter (sys.executable) will be used
  52. to run the command.
  53. Example:
  54. hooks = [
  55. { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
  56. "action": ["python", "image_indexer.py", "--all"]},
  57. ]
  58. """
  59. __author__ = "darinf@gmail.com (Darin Fisher)"
  60. __version__ = "0.3.1"
  61. import errno
  62. import optparse
  63. import os
  64. import re
  65. import stat
  66. import subprocess
  67. import sys
  68. import time
  69. import urlparse
  70. import xml.dom.minidom
  71. import urllib
  72. def getText(nodelist):
  73. """
  74. Return the concatenated text for the children of a list of DOM nodes.
  75. """
  76. rc = []
  77. for node in nodelist:
  78. if node.nodeType == node.TEXT_NODE:
  79. rc.append(node.data)
  80. else:
  81. rc.append(getText(node.childNodes))
  82. return ''.join(rc)
  83. SVN_COMMAND = "svn"
  84. # default help text
  85. DEFAULT_USAGE_TEXT = (
  86. """usage: %prog <subcommand> [options] [--] [svn options/args...]
  87. a wrapper for managing a set of client modules in svn.
  88. Version """ + __version__ + """
  89. subcommands:
  90. cleanup
  91. config
  92. diff
  93. revert
  94. status
  95. sync
  96. update
  97. runhooks
  98. revinfo
  99. Options and extra arguments can be passed to invoked svn commands by
  100. appending them to the command line. Note that if the first such
  101. appended option starts with a dash (-) then the options must be
  102. preceded by -- to distinguish them from gclient options.
  103. For additional help on a subcommand or examples of usage, try
  104. %prog help <subcommand>
  105. %prog help files
  106. """)
  107. GENERIC_UPDATE_USAGE_TEXT = (
  108. """Perform a checkout/update of the modules specified by the gclient
  109. configuration; see 'help config'. Unless --revision is specified,
  110. then the latest revision of the root solutions is checked out, with
  111. dependent submodule versions updated according to DEPS files.
  112. If --revision is specified, then the given revision is used in place
  113. of the latest, either for a single solution or for all solutions.
  114. Unless the --force option is provided, solutions and modules whose
  115. local revision matches the one to update (i.e., they have not changed
  116. in the repository) are *not* modified.
  117. This a synonym for 'gclient %(alias)s'
  118. usage: gclient %(cmd)s [options] [--] [svn update options/args]
  119. Valid options:
  120. --force : force update even for unchanged modules
  121. --revision REV : update/checkout all solutions with specified revision
  122. --revision SOLUTION@REV : update given solution to specified revision
  123. --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
  124. --verbose : output additional diagnostics
  125. Examples:
  126. gclient %(cmd)s
  127. update files from SVN according to current configuration,
  128. *for modules which have changed since last update or sync*
  129. gclient %(cmd)s --force
  130. update files from SVN according to current configuration, for
  131. all modules (useful for recovering files deleted from local copy)
  132. """)
  133. COMMAND_USAGE_TEXT = {
  134. "cleanup":
  135. """Clean up all working copies, using 'svn cleanup' for each module.
  136. Additional options and args may be passed to 'svn cleanup'.
  137. usage: cleanup [options] [--] [svn cleanup args/options]
  138. Valid options:
  139. --verbose : output additional diagnostics
  140. """,
  141. "config": """Create a .gclient file in the current directory; this
  142. specifies the configuration for further commands. After update/sync,
  143. top-level DEPS files in each module are read to determine dependent
  144. modules to operate on as well. If optional [url] parameter is
  145. provided, then configuration is read from a specified Subversion server
  146. URL. Otherwise, a --spec option must be provided.
  147. usage: config [option | url] [safesync url]
  148. Valid options:
  149. --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
  150. *Note that due to Cygwin/Python brokenness, it
  151. probably can't contain any newlines.*
  152. Examples:
  153. gclient config https://gclient.googlecode.com/svn/trunk/gclient
  154. configure a new client to check out gclient.py tool sources
  155. gclient config --spec='solutions=[{"name":"gclient","""
  156. '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
  157. '"custom_deps":{}}]',
  158. "diff": """Display the differences between two revisions of modules.
  159. (Does 'svn diff' for each checked out module and dependences.)
  160. Additional args and options to 'svn diff' can be passed after
  161. gclient options.
  162. usage: diff [options] [--] [svn args/options]
  163. Valid options:
  164. --verbose : output additional diagnostics
  165. Examples:
  166. gclient diff
  167. simple 'svn diff' for configured client and dependences
  168. gclient diff -- -x -b
  169. use 'svn diff -x -b' to suppress whitespace-only differences
  170. gclient diff -- -r HEAD -x -b
  171. diff versus the latest version of each module
  172. """,
  173. "revert":
  174. """Revert every file in every managed directory in the client view.
  175. usage: revert
  176. """,
  177. "status":
  178. """Show the status of client and dependent modules, using 'svn diff'
  179. for each module. Additional options and args may be passed to 'svn diff'.
  180. usage: status [options] [--] [svn diff args/options]
  181. Valid options:
  182. --verbose : output additional diagnostics
  183. """,
  184. "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
  185. "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
  186. "help": """Describe the usage of this program or its subcommands.
  187. usage: help [options] [subcommand]
  188. Valid options:
  189. --verbose : output additional diagnostics
  190. """,
  191. "runhooks":
  192. """Runs hooks for files that have been modified in the local working copy,
  193. according to 'svn status'.
  194. usage: runhooks [options]
  195. Valid options:
  196. --force : runs all known hooks, regardless of the working
  197. copy status
  198. --verbose : output additional diagnostics
  199. """,
  200. "revinfo":
  201. """Outputs source path, server URL and revision information for every
  202. dependency in all solutions (no local checkout required).
  203. usage: revinfo [options]
  204. """,
  205. }
  206. # parameterized by (solution_name, solution_url, safesync_url)
  207. DEFAULT_CLIENT_FILE_TEXT = (
  208. """
  209. # An element of this array (a \"solution\") describes a repository directory
  210. # that will be checked out into your working copy. Each solution may
  211. # optionally define additional dependencies (via its DEPS file) to be
  212. # checked out alongside the solution's directory. A solution may also
  213. # specify custom dependencies (via the \"custom_deps\" property) that
  214. # override or augment the dependencies specified by the DEPS file.
  215. # If a \"safesync_url\" is specified, it is assumed to reference the location of
  216. # a text file which contains nothing but the last known good SCM revision to
  217. # sync against. It is fetched if specified and used unless --head is passed
  218. solutions = [
  219. { \"name\" : \"%s\",
  220. \"url\" : \"%s\",
  221. \"custom_deps\" : {
  222. # To use the trunk of a component instead of what's in DEPS:
  223. #\"component\": \"https://svnserver/component/trunk/\",
  224. # To exclude a component from your working copy:
  225. #\"data/really_large_component\": None,
  226. },
  227. \"safesync_url\": \"%s\"
  228. }
  229. ]
  230. """)
  231. ## Generic utils
  232. class Error(Exception):
  233. """gclient exception class."""
  234. pass
  235. class PrintableObject(object):
  236. def __str__(self):
  237. output = ''
  238. for i in dir(self):
  239. if i.startswith('__'):
  240. continue
  241. output += '%s = %s\n' % (i, str(getattr(self, i, '')))
  242. return output
  243. def FileRead(filename):
  244. content = None
  245. f = open(filename, "rU")
  246. try:
  247. content = f.read()
  248. finally:
  249. f.close()
  250. return content
  251. def FileWrite(filename, content):
  252. f = open(filename, "w")
  253. try:
  254. f.write(content)
  255. finally:
  256. f.close()
  257. def RemoveDirectory(*path):
  258. """Recursively removes a directory, even if it's marked read-only.
  259. Remove the directory located at *path, if it exists.
  260. shutil.rmtree() doesn't work on Windows if any of the files or directories
  261. are read-only, which svn repositories and some .svn files are. We need to
  262. be able to force the files to be writable (i.e., deletable) as we traverse
  263. the tree.
  264. Even with all this, Windows still sometimes fails to delete a file, citing
  265. a permission error (maybe something to do with antivirus scans or disk
  266. indexing). The best suggestion any of the user forums had was to wait a
  267. bit and try again, so we do that too. It's hand-waving, but sometimes it
  268. works. :/
  269. On POSIX systems, things are a little bit simpler. The modes of the files
  270. to be deleted doesn't matter, only the modes of the directories containing
  271. them are significant. As the directory tree is traversed, each directory
  272. has its mode set appropriately before descending into it. This should
  273. result in the entire tree being removed, with the possible exception of
  274. *path itself, because nothing attempts to change the mode of its parent.
  275. Doing so would be hazardous, as it's not a directory slated for removal.
  276. In the ordinary case, this is not a problem: for our purposes, the user
  277. will never lack write permission on *path's parent.
  278. """
  279. file_path = os.path.join(*path)
  280. if not os.path.exists(file_path):
  281. return
  282. if os.path.islink(file_path) or not os.path.isdir(file_path):
  283. raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
  284. has_win32api = False
  285. if sys.platform == 'win32':
  286. has_win32api = True
  287. # Some people don't have the APIs installed. In that case we'll do without.
  288. try:
  289. win32api = __import__('win32api')
  290. win32con = __import__('win32con')
  291. except ImportError:
  292. has_win32api = False
  293. else:
  294. # On POSIX systems, we need the x-bit set on the directory to access it,
  295. # the r-bit to see its contents, and the w-bit to remove files from it.
  296. # The actual modes of the files within the directory is irrelevant.
  297. os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
  298. for fn in os.listdir(file_path):
  299. fullpath = os.path.join(file_path, fn)
  300. # If fullpath is a symbolic link that points to a directory, isdir will
  301. # be True, but we don't want to descend into that as a directory, we just
  302. # want to remove the link. Check islink and treat links as ordinary files
  303. # would be treated regardless of what they reference.
  304. if os.path.islink(fullpath) or not os.path.isdir(fullpath):
  305. if sys.platform == 'win32':
  306. os.chmod(fullpath, stat.S_IWRITE)
  307. if has_win32api:
  308. win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
  309. try:
  310. os.remove(fullpath)
  311. except OSError, e:
  312. if e.errno != errno.EACCES or sys.platform != 'win32':
  313. raise
  314. print 'Failed to delete %s: trying again' % fullpath
  315. time.sleep(0.1)
  316. os.remove(fullpath)
  317. else:
  318. RemoveDirectory(fullpath)
  319. if sys.platform == 'win32':
  320. os.chmod(file_path, stat.S_IWRITE)
  321. if has_win32api:
  322. win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
  323. try:
  324. os.rmdir(file_path)
  325. except OSError, e:
  326. if e.errno != errno.EACCES or sys.platform != 'win32':
  327. raise
  328. print 'Failed to remove %s: trying again' % file_path
  329. time.sleep(0.1)
  330. os.rmdir(file_path)
  331. def SubprocessCall(command, in_directory, out, fail_status=None):
  332. """Runs command, a list, in directory in_directory.
  333. This function wraps SubprocessCallAndCapture, but does not perform the
  334. capturing functions. See that function for a more complete usage
  335. description.
  336. """
  337. # Call subprocess and capture nothing:
  338. SubprocessCallAndCapture(command, in_directory, out, fail_status)
  339. def SubprocessCallAndCapture(command, in_directory, out, fail_status=None,
  340. pattern=None, capture_list=None):
  341. """Runs command, a list, in directory in_directory.
  342. A message indicating what is being done, as well as the command's stdout,
  343. is printed to out.
  344. If a pattern is specified, any line in the output matching pattern will have
  345. its first match group appended to capture_list.
  346. If the command fails, as indicated by a nonzero exit status, gclient will
  347. exit with an exit status of fail_status. If fail_status is None (the
  348. default), gclient will raise an Error exception.
  349. """
  350. print >> out, ("\n________ running \'%s\' in \'%s\'"
  351. % (' '.join(command), in_directory))
  352. # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
  353. # executable, but shell=True makes subprocess on Linux fail when it's called
  354. # with a list because it only tries to execute the first item in the list.
  355. kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
  356. shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
  357. if pattern:
  358. compiled_pattern = re.compile(pattern)
  359. # Also, we need to forward stdout to prevent weird re-ordering of output.
  360. # This has to be done on a per byte basis to make sure it is not buffered:
  361. # normally buffering is done for each line, but if svn requests input, no
  362. # end-of-line character is output after the prompt and it would not show up.
  363. in_byte = kid.stdout.read(1)
  364. in_line = ""
  365. while in_byte:
  366. if in_byte != "\r":
  367. out.write(in_byte)
  368. in_line += in_byte
  369. if in_byte == "\n" and pattern:
  370. match = compiled_pattern.search(in_line[:-1])
  371. if match:
  372. capture_list.append(match.group(1))
  373. in_line = ""
  374. in_byte = kid.stdout.read(1)
  375. rv = kid.wait()
  376. if rv:
  377. msg = "failed to run command: %s" % " ".join(command)
  378. if fail_status != None:
  379. print >>sys.stderr, msg
  380. sys.exit(fail_status)
  381. raise Error(msg)
  382. def IsUsingGit(root, paths):
  383. """Returns True if we're using git to manage any of our checkouts.
  384. |entries| is a list of paths to check."""
  385. for path in paths:
  386. if os.path.exists(os.path.join(root, path, '.git')):
  387. return True
  388. return False
  389. # -----------------------------------------------------------------------------
  390. # SVN utils:
  391. def RunSVN(options, args, in_directory):
  392. """Runs svn, sending output to stdout.
  393. Args:
  394. args: A sequence of command line parameters to be passed to svn.
  395. in_directory: The directory where svn is to be run.
  396. Raises:
  397. Error: An error occurred while running the svn command.
  398. """
  399. c = [SVN_COMMAND]
  400. c.extend(args)
  401. SubprocessCall(c, in_directory, options.stdout)
  402. def CaptureSVN(options, args, in_directory):
  403. """Runs svn, capturing output sent to stdout as a string.
  404. Args:
  405. args: A sequence of command line parameters to be passed to svn.
  406. in_directory: The directory where svn is to be run.
  407. Returns:
  408. The output sent to stdout as a string.
  409. """
  410. c = [SVN_COMMAND]
  411. c.extend(args)
  412. # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
  413. # the svn.exe executable, but shell=True makes subprocess on Linux fail
  414. # when it's called with a list because it only tries to execute the
  415. # first string ("svn").
  416. return subprocess.Popen(c, cwd=in_directory, shell=(sys.platform == 'win32'),
  417. stdout=subprocess.PIPE).communicate()[0]
  418. def RunSVNAndGetFileList(options, args, in_directory, file_list):
  419. """Runs svn checkout, update, or status, output to stdout.
  420. The first item in args must be either "checkout", "update", or "status".
  421. svn's stdout is parsed to collect a list of files checked out or updated.
  422. These files are appended to file_list. svn's stdout is also printed to
  423. sys.stdout as in RunSVN.
  424. Args:
  425. args: A sequence of command line parameters to be passed to svn.
  426. in_directory: The directory where svn is to be run.
  427. Raises:
  428. Error: An error occurred while running the svn command.
  429. """
  430. command = [SVN_COMMAND]
  431. command.extend(args)
  432. # svn update and svn checkout use the same pattern: the first three columns
  433. # are for file status, property status, and lock status. This is followed
  434. # by two spaces, and then the path to the file.
  435. update_pattern = '^... (.*)$'
  436. # The first three columns of svn status are the same as for svn update and
  437. # svn checkout. The next three columns indicate addition-with-history,
  438. # switch, and remote lock status. This is followed by one space, and then
  439. # the path to the file.
  440. status_pattern = '^...... (.*)$'
  441. # args[0] must be a supported command. This will blow up if it's something
  442. # else, which is good. Note that the patterns are only effective when
  443. # these commands are used in their ordinary forms, the patterns are invalid
  444. # for "svn status --show-updates", for example.
  445. pattern = {
  446. 'checkout': update_pattern,
  447. 'status': status_pattern,
  448. 'update': update_pattern,
  449. }[args[0]]
  450. SubprocessCallAndCapture(command, in_directory, options.stdout,
  451. pattern=pattern, capture_list=file_list)
  452. def CaptureSVNInfo(options, relpath, in_directory):
  453. """Runs 'svn info' on an existing path.
  454. Args:
  455. relpath: The directory where the working copy resides relative to
  456. the directory given by in_directory.
  457. in_directory: The directory where svn is to be run.
  458. Returns:
  459. An object with fields corresponding to the output of 'svn info'
  460. """
  461. info = CaptureSVN(options, ["info", "--xml", relpath], in_directory)
  462. dom = xml.dom.minidom.parseString(info)
  463. # str() the getText() results because they may be returned as
  464. # Unicode, which interferes with the higher layers matching up
  465. # things in the deps dictionary.
  466. result = PrintableObject()
  467. result.root = str(getText(dom.getElementsByTagName('root')))
  468. result.url = str(getText(dom.getElementsByTagName('url')))
  469. result.uuid = str(getText(dom.getElementsByTagName('uuid')))
  470. result.revision = int(dom.getElementsByTagName('entry')[0].getAttribute(
  471. 'revision'))
  472. return result
  473. def CaptureSVNHeadRevision(options, url):
  474. """Get the head revision of a SVN repository.
  475. Returns:
  476. Int head revision
  477. """
  478. info = CaptureSVN(options, ["info", "--xml", url], os.getcwd())
  479. dom = xml.dom.minidom.parseString(info)
  480. return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
  481. class FileStatus:
  482. def __init__(self, path, text_status, props, locked, history, switched,
  483. repo_locked, out_of_date):
  484. self.path = path.strip()
  485. self.text_status = text_status
  486. self.props = props
  487. self.locked = locked
  488. self.history = history
  489. self.switched = switched
  490. self.repo_locked = repo_locked
  491. self.out_of_date = out_of_date
  492. def __str__(self):
  493. return (self.text_status + self.props + self.locked + self.history +
  494. self.switched + self.repo_locked + self.out_of_date +
  495. self.path)
  496. def CaptureSVNStatus(options, path):
  497. """Runs 'svn status' on an existing path.
  498. Args:
  499. path: The directory to run svn status.
  500. Returns:
  501. An array of FileStatus corresponding to the output of 'svn status'
  502. """
  503. info = CaptureSVN(options, ["status"], path)
  504. result = []
  505. if not info:
  506. return result
  507. for line in info.splitlines():
  508. if line:
  509. new_item = FileStatus(line[7:], line[0:1], line[1:2], line[2:3],
  510. line[3:4], line[4:5], line[5:6], line[6:7])
  511. result.append(new_item)
  512. return result
  513. ### SCM abstraction layer
  514. class SCMWrapper(object):
  515. """Add necessary glue between all the supported SCM.
  516. This is the abstraction layer to bind to different SCM. Since currently only
  517. subversion is supported, a lot of subersionism remains. This can be sorted out
  518. once another SCM is supported."""
  519. def __init__(self, url=None, root_dir=None, relpath=None,
  520. scm_name='svn'):
  521. # TODO(maruel): Deduce the SCM from the url.
  522. self.scm_name = scm_name
  523. self.url = url
  524. self._root_dir = root_dir
  525. if self._root_dir:
  526. self._root_dir = self._root_dir.replace('/', os.sep).strip()
  527. self.relpath = relpath
  528. if self.relpath:
  529. self.relpath = self.relpath.replace('/', os.sep).strip()
  530. def FullUrlForRelativeUrl(self, url):
  531. # Find the forth '/' and strip from there. A bit hackish.
  532. return '/'.join(self.url.split('/')[:4]) + url
  533. def RunCommand(self, command, options, args, file_list=None):
  534. # file_list will have all files that are modified appended to it.
  535. if file_list == None:
  536. file_list = []
  537. commands = {
  538. 'cleanup': self.cleanup,
  539. 'update': self.update,
  540. 'revert': self.revert,
  541. 'status': self.status,
  542. 'diff': self.diff,
  543. 'runhooks': self.status,
  544. }
  545. if not command in commands:
  546. raise Error('Unknown command %s' % command)
  547. return commands[command](options, args, file_list)
  548. def cleanup(self, options, args, file_list):
  549. """Cleanup working copy."""
  550. command = ['cleanup']
  551. command.extend(args)
  552. RunSVN(options, command, os.path.join(self._root_dir, self.relpath))
  553. def diff(self, options, args, file_list):
  554. # NOTE: This function does not currently modify file_list.
  555. command = ['diff']
  556. command.extend(args)
  557. RunSVN(options, command, os.path.join(self._root_dir, self.relpath))
  558. def update(self, options, args, file_list):
  559. """Runs SCM to update or transparently checkout the working copy.
  560. All updated files will be appended to file_list.
  561. Raises:
  562. Error: if can't get URL for relative path.
  563. """
  564. # Only update if git is not controlling the directory.
  565. git_path = os.path.join(self._root_dir, self.relpath, '.git')
  566. if options.path_exists(git_path):
  567. print >> options.stdout, (
  568. "________ found .git directory; skipping %s" % self.relpath)
  569. return
  570. if args:
  571. raise Error("Unsupported argument(s): %s" % ",".join(args))
  572. url = self.url
  573. components = url.split("@")
  574. revision = None
  575. forced_revision = False
  576. if options.revision:
  577. # Override the revision number.
  578. url = '%s@%s' % (components[0], str(options.revision))
  579. revision = int(options.revision)
  580. forced_revision = True
  581. elif len(components) == 2:
  582. revision = int(components[1])
  583. forced_revision = True
  584. rev_str = ""
  585. if revision:
  586. rev_str = ' at %d' % revision
  587. if not options.path_exists(os.path.join(self._root_dir, self.relpath)):
  588. # We need to checkout.
  589. command = ['checkout', url, os.path.join(self._root_dir, self.relpath)]
  590. RunSVNAndGetFileList(options, command, self._root_dir, file_list)
  591. # Get the existing scm url and the revision number of the current checkout.
  592. from_info = CaptureSVNInfo(options,
  593. os.path.join(self._root_dir, self.relpath, '.'),
  594. '.')
  595. if options.manually_grab_svn_rev:
  596. # Retrieve the current HEAD version because svn is slow at null updates.
  597. if not revision:
  598. from_info_live = CaptureSVNInfo(options, from_info.url, '.')
  599. revision = int(from_info_live.revision)
  600. rev_str = ' at %d' % revision
  601. if from_info.url != components[0]:
  602. to_info = CaptureSVNInfo(options, url, '.')
  603. if from_info.root != to_info.root:
  604. # We have different roots, so check if we can switch --relocate.
  605. # Subversion only permits this if the repository UUIDs match.
  606. if from_info.uuid != to_info.uuid:
  607. raise Error("Can't switch the checkout to %s; UUID don't match" % url)
  608. # Perform the switch --relocate, then rewrite the from_url
  609. # to reflect where we "are now." (This is the same way that
  610. # Subversion itself handles the metadata when switch --relocate
  611. # is used.) This makes the checks below for whether we
  612. # can update to a revision or have to switch to a different
  613. # branch work as expected.
  614. # TODO(maruel): TEST ME !
  615. command = ["switch", "--relocate", from_info.root, to_info.root,
  616. self.relpath]
  617. RunSVN(options, command, self._root_dir)
  618. from_info.url = from_info.url.replace(from_info.root, to_info.root)
  619. # If the provided url has a revision number that matches the revision
  620. # number of the existing directory, then we don't need to bother updating.
  621. if not options.force and from_info.revision == revision:
  622. if options.verbose or not forced_revision:
  623. print >>options.stdout, ("\n_____ %s%s" % (
  624. self.relpath, rev_str))
  625. return
  626. command = ["update", os.path.join(self._root_dir, self.relpath)]
  627. if revision:
  628. command.extend(['--revision', str(revision)])
  629. RunSVNAndGetFileList(options, command, self._root_dir, file_list)
  630. def revert(self, options, args, file_list):
  631. """Reverts local modifications. Subversion specific.
  632. All reverted files will be appended to file_list, even if Subversion
  633. doesn't know about them.
  634. """
  635. path = os.path.join(self._root_dir, self.relpath)
  636. if not os.path.isdir(path):
  637. # We can't revert path that doesn't exist.
  638. # TODO(maruel): Should we update instead?
  639. if options.verbose:
  640. print >>options.stdout, ("\n_____ %s is missing, can't revert" %
  641. self.relpath)
  642. return
  643. files = CaptureSVNStatus(options, path)
  644. # Batch the command.
  645. files_to_revert = []
  646. for file in files:
  647. file_path = os.path.join(path, file.path)
  648. print >>options.stdout, file_path
  649. # Unversioned file or unexpected unversioned file.
  650. if file.text_status in ('?', '~'):
  651. # Remove extraneous file. Also remove unexpected unversioned
  652. # directories. svn won't touch them but we want to delete these.
  653. file_list.append(file_path)
  654. try:
  655. os.remove(file_path)
  656. except EnvironmentError:
  657. RemoveDirectory(file_path)
  658. if file.text_status != '?':
  659. # For any other status, svn revert will work.
  660. file_list.append(file_path)
  661. files_to_revert.append(file.path)
  662. # Revert them all at once.
  663. if files_to_revert:
  664. accumulated_paths = []
  665. accumulated_length = 0
  666. command = ['revert']
  667. for p in files_to_revert:
  668. # Some shell have issues with command lines too long.
  669. if accumulated_length and accumulated_length + len(p) > 3072:
  670. RunSVN(options, command + accumulated_paths,
  671. os.path.join(self._root_dir, self.relpath))
  672. accumulated_paths = []
  673. accumulated_length = 0
  674. else:
  675. accumulated_paths.append(p)
  676. accumulated_length += len(p)
  677. if accumulated_paths:
  678. RunSVN(options, command + accumulated_paths,
  679. os.path.join(self._root_dir, self.relpath))
  680. def status(self, options, args, file_list):
  681. """Display status information."""
  682. command = ['status']
  683. command.extend(args)
  684. RunSVNAndGetFileList(options, command,
  685. os.path.join(self._root_dir, self.relpath), file_list)
  686. ## GClient implementation.
  687. class GClient(object):
  688. """Object that represent a gclient checkout."""
  689. supported_commands = [
  690. 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
  691. ]
  692. def __init__(self, root_dir, options):
  693. self._root_dir = root_dir
  694. self._options = options
  695. self._config_content = None
  696. self._config_dict = {}
  697. self._deps_hooks = []
  698. def SetConfig(self, content):
  699. self._config_dict = {}
  700. self._config_content = content
  701. exec(content, self._config_dict)
  702. def SaveConfig(self):
  703. FileWrite(os.path.join(self._root_dir, self._options.config_filename),
  704. self._config_content)
  705. def _LoadConfig(self):
  706. client_source = FileRead(os.path.join(self._root_dir,
  707. self._options.config_filename))
  708. self.SetConfig(client_source)
  709. def ConfigContent(self):
  710. return self._config_content
  711. def GetVar(self, key, default=None):
  712. return self._config_dict.get(key, default)
  713. @staticmethod
  714. def LoadCurrentConfig(options, from_dir=None):
  715. """Searches for and loads a .gclient file relative to the current working
  716. dir.
  717. Returns:
  718. A dict representing the contents of the .gclient file or an empty dict if
  719. the .gclient file doesn't exist.
  720. """
  721. if not from_dir:
  722. from_dir = os.curdir
  723. path = os.path.realpath(from_dir)
  724. while not options.path_exists(os.path.join(path, options.config_filename)):
  725. next = os.path.split(path)
  726. if not next[1]:
  727. return None
  728. path = next[0]
  729. client = options.gclient(path, options)
  730. client._LoadConfig()
  731. return client
  732. def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
  733. self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
  734. solution_name, solution_url, safesync_url
  735. ))
  736. def _SaveEntries(self, entries):
  737. """Creates a .gclient_entries file to record the list of unique checkouts.
  738. The .gclient_entries file lives in the same directory as .gclient.
  739. Args:
  740. entries: A sequence of solution names.
  741. """
  742. text = "entries = [\n"
  743. for entry in entries:
  744. text += " \"%s\",\n" % entry
  745. text += "]\n"
  746. FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
  747. text)
  748. def _ReadEntries(self):
  749. """Read the .gclient_entries file for the given client.
  750. Args:
  751. client: The client for which the entries file should be read.
  752. Returns:
  753. A sequence of solution names, which will be empty if there is the
  754. entries file hasn't been created yet.
  755. """
  756. scope = {}
  757. filename = os.path.join(self._root_dir, self._options.entries_filename)
  758. if not self._options.path_exists(filename):
  759. return []
  760. exec(FileRead(filename), scope)
  761. return scope["entries"]
  762. class FromImpl:
  763. """Used to implement the From syntax."""
  764. def __init__(self, module_name):
  765. self.module_name = module_name
  766. def __str__(self):
  767. return 'From("%s")' % self.module_name
  768. class _VarImpl:
  769. def __init__(self, custom_vars, local_scope):
  770. self._custom_vars = custom_vars
  771. self._local_scope = local_scope
  772. def Lookup(self, var_name):
  773. """Implements the Var syntax."""
  774. if var_name in self._custom_vars:
  775. return self._custom_vars[var_name]
  776. elif var_name in self._local_scope.get("vars", {}):
  777. return self._local_scope["vars"][var_name]
  778. raise Error("Var is not defined: %s" % var_name)
  779. def _ParseSolutionDeps(self, solution_name, solution_deps_content,
  780. custom_vars):
  781. """Parses the DEPS file for the specified solution.
  782. Args:
  783. solution_name: The name of the solution to query.
  784. solution_deps_content: Content of the DEPS file for the solution
  785. custom_vars: A dict of vars to override any vars defined in the DEPS file.
  786. Returns:
  787. A dict mapping module names (as relative paths) to URLs or an empty
  788. dict if the solution does not have a DEPS file.
  789. """
  790. # Skip empty
  791. if not solution_deps_content:
  792. return {}
  793. # Eval the content
  794. local_scope = {}
  795. var = self._VarImpl(custom_vars, local_scope)
  796. global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
  797. exec(solution_deps_content, global_scope, local_scope)
  798. deps = local_scope.get("deps", {})
  799. # load os specific dependencies if defined. these dependencies may
  800. # override or extend the values defined by the 'deps' member.
  801. if "deps_os" in local_scope:
  802. deps_os_choices = {
  803. "win32": "win",
  804. "win": "win",
  805. "cygwin": "win",
  806. "darwin": "mac",
  807. "mac": "mac",
  808. "unix": "unix",
  809. "linux": "unix",
  810. "linux2": "unix",
  811. }
  812. if self._options.deps_os is not None:
  813. deps_to_include = self._options.deps_os.split(",")
  814. if "all" in deps_to_include:
  815. deps_to_include = deps_os_choices.values()
  816. else:
  817. deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
  818. deps_to_include = set(deps_to_include)
  819. for deps_os_key in deps_to_include:
  820. os_deps = local_scope["deps_os"].get(deps_os_key, {})
  821. if len(deps_to_include) > 1:
  822. # Ignore any overrides when including deps for more than one
  823. # platform, so we collect the broadest set of dependencies available.
  824. # We may end up with the wrong revision of something for our
  825. # platform, but this is the best we can do.
  826. deps.update([x for x in os_deps.items() if not x[0] in deps])
  827. else:
  828. deps.update(os_deps)
  829. if 'hooks' in local_scope:
  830. self._deps_hooks.extend(local_scope['hooks'])
  831. # If use_relative_paths is set in the DEPS file, regenerate
  832. # the dictionary using paths relative to the directory containing
  833. # the DEPS file.
  834. if local_scope.get('use_relative_paths'):
  835. rel_deps = {}
  836. for d, url in deps.items():
  837. # normpath is required to allow DEPS to use .. in their
  838. # dependency local path.
  839. rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
  840. return rel_deps
  841. else:
  842. return deps
  843. def _ParseAllDeps(self, solution_urls, solution_deps_content):
  844. """Parse the complete list of dependencies for the client.
  845. Args:
  846. solution_urls: A dict mapping module names (as relative paths) to URLs
  847. corresponding to the solutions specified by the client. This parameter
  848. is passed as an optimization.
  849. solution_deps_content: A dict mapping module names to the content
  850. of their DEPS files
  851. Returns:
  852. A dict mapping module names (as relative paths) to URLs corresponding
  853. to the entire set of dependencies to checkout for the given client.
  854. Raises:
  855. Error: If a dependency conflicts with another dependency or of a solution.
  856. """
  857. deps = {}
  858. for solution in self.GetVar("solutions"):
  859. custom_vars = solution.get("custom_vars", {})
  860. solution_deps = self._ParseSolutionDeps(
  861. solution["name"],
  862. solution_deps_content[solution["name"]],
  863. custom_vars)
  864. # If a line is in custom_deps, but not in the solution, we want to append
  865. # this line to the solution.
  866. if "custom_deps" in solution:
  867. for d in solution["custom_deps"]:
  868. if d not in solution_deps:
  869. solution_deps[d] = solution["custom_deps"][d]
  870. for d in solution_deps:
  871. if "custom_deps" in solution and d in solution["custom_deps"]:
  872. # Dependency is overriden.
  873. url = solution["custom_deps"][d]
  874. if url is None:
  875. continue
  876. else:
  877. url = solution_deps[d]
  878. # if we have a From reference dependent on another solution, then
  879. # just skip the From reference. When we pull deps for the solution,
  880. # we will take care of this dependency.
  881. #
  882. # If multiple solutions all have the same From reference, then we
  883. # should only add one to our list of dependencies.
  884. if type(url) != str:
  885. if url.module_name in solution_urls:
  886. # Already parsed.
  887. continue
  888. if d in deps and type(deps[d]) != str:
  889. if url.module_name == deps[d].module_name:
  890. continue
  891. else:
  892. parsed_url = urlparse.urlparse(url)
  893. scheme = parsed_url[0]
  894. if not scheme:
  895. # A relative url. Fetch the real base.
  896. path = parsed_url[2]
  897. if path[0] != "/":
  898. raise Error(
  899. "relative DEPS entry \"%s\" must begin with a slash" % d)
  900. # Create a scm just to query the full url.
  901. scm = self._options.scm_wrapper(solution["url"], self._root_dir,
  902. None)
  903. url = scm.FullUrlForRelativeUrl(url)
  904. if d in deps and deps[d] != url:
  905. raise Error(
  906. "Solutions have conflicting versions of dependency \"%s\"" % d)
  907. if d in solution_urls and solution_urls[d] != url:
  908. raise Error(
  909. "Dependency \"%s\" conflicts with specified solution" % d)
  910. # Grab the dependency.
  911. deps[d] = url
  912. return deps
  913. def _RunHookAction(self, hook_dict):
  914. """Runs the action from a single hook.
  915. """
  916. command = hook_dict['action'][:]
  917. if command[0] == 'python':
  918. # If the hook specified "python" as the first item, the action is a
  919. # Python script. Run it by starting a new copy of the same
  920. # interpreter.
  921. command[0] = sys.executable
  922. # Use a discrete exit status code of 2 to indicate that a hook action
  923. # failed. Users of this script may wish to treat hook action failures
  924. # differently from VC failures.
  925. SubprocessCall(command, self._root_dir, self._options.stdout,
  926. fail_status=2)
  927. def _RunHooks(self, command, file_list, is_using_git):
  928. """Evaluates all hooks, running actions as needed.
  929. """
  930. # Hooks only run for these command types.
  931. if not command in ('update', 'revert', 'runhooks'):
  932. return
  933. # Get any hooks from the .gclient file.
  934. hooks = self.GetVar("hooks", [])
  935. # Add any hooks found in DEPS files.
  936. hooks.extend(self._deps_hooks)
  937. # If "--force" was specified, run all hooks regardless of what files have
  938. # changed. If the user is using git, then we don't know what files have
  939. # changed so we always run all hooks.
  940. if self._options.force or is_using_git:
  941. for hook_dict in hooks:
  942. self._RunHookAction(hook_dict)
  943. return
  944. # Run hooks on the basis of whether the files from the gclient operation
  945. # match each hook's pattern.
  946. for hook_dict in hooks:
  947. pattern = re.compile(hook_dict['pattern'])
  948. for file in file_list:
  949. if not pattern.search(file):
  950. continue
  951. self._RunHookAction(hook_dict)
  952. # The hook's action only runs once. Don't bother looking for any
  953. # more matches.
  954. break
  955. def RunOnDeps(self, command, args):
  956. """Runs a command on each dependency in a client and its dependencies.
  957. The module's dependencies are specified in its top-level DEPS files.
  958. Args:
  959. command: The command to use (e.g., 'status' or 'diff')
  960. args: list of str - extra arguments to add to the command line.
  961. Raises:
  962. Error: If the client has conflicting entries.
  963. """
  964. if not command in self.supported_commands:
  965. raise Error("'%s' is an unsupported command" % command)
  966. # Check for revision overrides.
  967. revision_overrides = {}
  968. for revision in self._options.revisions:
  969. if revision.find("@") == -1:
  970. raise Error(
  971. "Specify the full dependency when specifying a revision number.")
  972. revision_elem = revision.split("@")
  973. # Disallow conflicting revs
  974. if revision_overrides.has_key(revision_elem[0]) and \
  975. revision_overrides[revision_elem[0]] != revision_elem[1]:
  976. raise Error(
  977. "Conflicting revision numbers specified.")
  978. revision_overrides[revision_elem[0]] = revision_elem[1]
  979. solutions = self.GetVar("solutions")
  980. if not solutions:
  981. raise Error("No solution specified")
  982. # When running runhooks --force, there's no need to consult the SCM.
  983. # All known hooks are expected to run unconditionally regardless of working
  984. # copy state, so skip the SCM status check.
  985. run_scm = not (command == 'runhooks' and self._options.force)
  986. entries = {}
  987. entries_deps_content = {}
  988. file_list = []
  989. # Run on the base solutions first.
  990. for solution in solutions:
  991. name = solution["name"]
  992. if name in entries:
  993. raise Error("solution %s specified more than once" % name)
  994. url = solution["url"]
  995. entries[name] = url
  996. if run_scm:
  997. self._options.revision = revision_overrides.get(name)
  998. scm = self._options.scm_wrapper(url, self._root_dir, name)
  999. scm.RunCommand(command, self._options, args, file_list)
  1000. self._options.revision = None
  1001. try:
  1002. deps_content = FileRead(os.path.join(self._root_dir, name,
  1003. self._options.deps_file))
  1004. except IOError, e:
  1005. if e.errno != errno.ENOENT:
  1006. raise
  1007. deps_content = ""
  1008. entries_deps_content[name] = deps_content
  1009. # Process the dependencies next (sort alphanumerically to ensure that
  1010. # containing directories get populated first and for readability)
  1011. deps = self._ParseAllDeps(entries, entries_deps_content)
  1012. deps_to_process = deps.keys()
  1013. deps_to_process.sort()
  1014. # First pass for direct dependencies.
  1015. for d in deps_to_process:
  1016. if type(deps[d]) == str:
  1017. url = deps[d]
  1018. entries[d] = url
  1019. if run_scm:
  1020. self._options.revision = revision_overrides.get(d)
  1021. scm = self._options.scm_wrapper(url, self._root_dir, d)
  1022. scm.RunCommand(command, self._options, args, file_list)
  1023. self._options.revision = None
  1024. # Second pass for inherited deps (via the From keyword)
  1025. for d in deps_to_process:
  1026. if type(deps[d]) != str:
  1027. sub_deps = self._ParseSolutionDeps(
  1028. deps[d].module_name,
  1029. FileRead(os.path.join(self._root_dir,
  1030. deps[d].module_name,
  1031. self._options.deps_file)),
  1032. {})
  1033. url = sub_deps[d]
  1034. entries[d] = url
  1035. if run_scm:
  1036. self._options.revision = revision_overrides.get(d)
  1037. scm = self._options.scm_wrapper(url, self._root_dir, d)
  1038. scm.RunCommand(command, self._options, args, file_list)
  1039. self._options.revision = None
  1040. is_using_git = IsUsingGit(self._root_dir, entries.keys())
  1041. self._RunHooks(command, file_list, is_using_git)
  1042. if command == 'update':
  1043. # notify the user if there is an orphaned entry in their working copy.
  1044. # TODO(darin): we should delete this directory manually if it doesn't
  1045. # have any changes in it.
  1046. prev_entries = self._ReadEntries()
  1047. for entry in prev_entries:
  1048. e_dir = os.path.join(self._root_dir, entry)
  1049. if entry not in entries and self._options.path_exists(e_dir):
  1050. if CaptureSVNStatus(self._options, e_dir):
  1051. # There are modified files in this entry
  1052. entries[entry] = None # Keep warning until removed.
  1053. print >> self._options.stdout, (
  1054. "\nWARNING: \"%s\" is no longer part of this client. "
  1055. "It is recommended that you manually remove it.\n") % entry
  1056. else:
  1057. # Delete the entry
  1058. print >> self._options.stdout, ("\n________ deleting \'%s\' " +
  1059. "in \'%s\'") % (entry, self._root_dir)
  1060. RemoveDirectory(e_dir)
  1061. # record the current list of entries for next time
  1062. self._SaveEntries(entries)
  1063. def PrintRevInfo(self):
  1064. """Output revision info mapping for the client and its dependencies. This
  1065. allows the capture of a overall "revision" for the source tree that can
  1066. be used to reproduce the same tree in the future. The actual output
  1067. contains enough information (source paths, svn server urls and revisions)
  1068. that it can be used either to generate external svn commands (without
  1069. gclient) or as input to gclient's --rev option (with some massaging of
  1070. the data).
  1071. NOTE: Unlike RunOnDeps this does not require a local checkout and is run
  1072. on the Pulse master. It MUST NOT execute hooks.
  1073. Raises:
  1074. Error: If the client has conflicting entries.
  1075. """
  1076. # Check for revision overrides.
  1077. revision_overrides = {}
  1078. for revision in self._options.revisions:
  1079. if revision.find("@") < 0:
  1080. raise Error(
  1081. "Specify the full dependency when specifying a revision number.")
  1082. revision_elem = revision.split("@")
  1083. # Disallow conflicting revs
  1084. if revision_overrides.has_key(revision_elem[0]) and \
  1085. revision_overrides[revision_elem[0]] != revision_elem[1]:
  1086. raise Error(
  1087. "Conflicting revision numbers specified.")
  1088. revision_overrides[revision_elem[0]] = revision_elem[1]
  1089. solutions = self.GetVar("solutions")
  1090. if not solutions:
  1091. raise Error("No solution specified")
  1092. entries = {}
  1093. entries_deps_content = {}
  1094. # Inner helper to generate base url and rev tuple (including honoring
  1095. # |revision_overrides|)
  1096. def GetURLAndRev(name, original_url):
  1097. if original_url.find("@") < 0:
  1098. if revision_overrides.has_key(name):
  1099. return (original_url, int(revision_overrides[name]))
  1100. else:
  1101. # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
  1102. return (original_url, CaptureSVNHeadRevision(self._options,
  1103. original_url))
  1104. else:
  1105. url_components = original_url.split("@")
  1106. if revision_overrides.has_key(name):
  1107. return (url_components[0], int(revision_overrides[name]))
  1108. else:
  1109. return (url_components[0], int(url_components[1]))
  1110. # Run on the base solutions first.
  1111. for solution in solutions:
  1112. name = solution["name"]
  1113. if name in entries:
  1114. raise Error("solution %s specified more than once" % name)
  1115. (url, rev) = GetURLAndRev(name, solution["url"])
  1116. entries[name] = "%s@%d" % (url, rev)
  1117. # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
  1118. entries_deps_content[name] = CaptureSVN(
  1119. self._options,
  1120. ["cat",
  1121. "%s/%s@%d" % (url,
  1122. self._options.deps_file,
  1123. rev)],
  1124. os.getcwd())
  1125. # Process the dependencies next (sort alphanumerically to ensure that
  1126. # containing directories get populated first and for readability)
  1127. deps = self._ParseAllDeps(entries, entries_deps_content)
  1128. deps_to_process = deps.keys()
  1129. deps_to_process.sort()
  1130. # First pass for direct dependencies.
  1131. for d in deps_to_process:
  1132. if type(deps[d]) == str:
  1133. (url, rev) = GetURLAndRev(d, deps[d])
  1134. entries[d] = "%s@%d" % (url, rev)
  1135. # Second pass for inherited deps (via the From keyword)
  1136. for d in deps_to_process:
  1137. if type(deps[d]) != str:
  1138. deps_parent_url = entries[deps[d].module_name]
  1139. if deps_parent_url.find("@") < 0:
  1140. raise Error("From %s missing revisioned url" % deps[d].module_name)
  1141. deps_parent_url_components = deps_parent_url.split("@")
  1142. # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
  1143. deps_parent_content = CaptureSVN(
  1144. self._options,
  1145. ["cat",
  1146. "%s/%s@%s" % (deps_parent_url_components[0],
  1147. self._options.deps_file,
  1148. deps_parent_url_components[1])],
  1149. os.getcwd())
  1150. sub_deps = self._ParseSolutionDeps(
  1151. deps[d].module_name,
  1152. FileRead(os.path.join(self._root_dir,
  1153. deps[d].module_name,
  1154. self._options.deps_file)),
  1155. {})
  1156. (url, rev) = GetURLAndRev(d, sub_deps[d])
  1157. entries[d] = "%s@%d" % (url, rev)
  1158. print ";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())])
  1159. ## gclient commands.
  1160. def DoCleanup(options, args):
  1161. """Handle the cleanup subcommand.
  1162. Raises:
  1163. Error: if client isn't configured properly.
  1164. """
  1165. client = options.gclient.LoadCurrentConfig(options)
  1166. if not client:
  1167. raise Error("client not configured; see 'gclient config'")
  1168. if options.verbose:
  1169. # Print out the .gclient file. This is longer than if we just printed the
  1170. # client dict, but more legible, and it might contain helpful comments.
  1171. print >>options.stdout, client.ConfigContent()
  1172. options.verbose = True
  1173. return client.RunOnDeps('cleanup', args)
  1174. def DoConfig(options, args):
  1175. """Handle the config subcommand.
  1176. Args:
  1177. options: If options.spec set, a string providing contents of config file.
  1178. args: The command line args. If spec is not set,
  1179. then args[0] is a string URL to get for config file.
  1180. Raises:
  1181. Error: on usage error
  1182. """
  1183. if len(args) < 1 and not options.spec:
  1184. raise Error("required argument missing; see 'gclient help config'")
  1185. if options.path_exists(options.config_filename):
  1186. raise Error("%s file already exists in the current directory" %
  1187. options.config_filename)
  1188. client = options.gclient('.', options)
  1189. if options.spec:
  1190. client.SetConfig(options.spec)
  1191. else:
  1192. # TODO(darin): it would be nice to be able to specify an alternate relpath
  1193. # for the given URL.
  1194. base_url = args[0]
  1195. name = args[0].split("/")[-1]
  1196. safesync_url = ""
  1197. if len(args) > 1:
  1198. safesync_url = args[1]
  1199. client.SetDefaultConfig(name, base_url, safesync_url)
  1200. client.SaveConfig()
  1201. def DoHelp(options, args):
  1202. """Handle the help subcommand giving help for another subcommand.
  1203. Raises:
  1204. Error: if the command is unknown.
  1205. """
  1206. if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
  1207. print >>options.stdout, COMMAND_USAGE_TEXT[args[0]]
  1208. else:
  1209. raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
  1210. def DoStatus(options, args):
  1211. """Handle the status subcommand.
  1212. Raises:
  1213. Error: if client isn't configured properly.
  1214. """
  1215. client = options.gclient.LoadCurrentConfig(options)
  1216. if not client:
  1217. raise Error("client not configured; see 'gclient config'")
  1218. if options.verbose:
  1219. # Print out the .gclient file. This is longer than if we just printed the
  1220. # client dict, but more legible, and it might contain helpful comments.
  1221. print >>options.stdout, client.ConfigContent()
  1222. options.verbose = True
  1223. return client.RunOnDeps('status', args)
  1224. def DoUpdate(options, args):
  1225. """Handle the update and sync subcommands.
  1226. Raises:
  1227. Error: if client isn't configured properly.
  1228. """
  1229. client = options.gclient.LoadCurrentConfig(options)
  1230. if not client:
  1231. raise Error("client not configured; see 'gclient config'")
  1232. if not options.head:
  1233. solutions = client.GetVar('solutions')
  1234. if solutions:
  1235. for s in solutions:
  1236. if s.get('safesync_url', ''):
  1237. # rip through revisions and make sure we're not over-riding
  1238. # something that was explicitly passed
  1239. has_key = False
  1240. for r in options.revisions:
  1241. if r.split('@')[0] == s['name']:
  1242. has_key = True
  1243. break
  1244. if not has_key:
  1245. handle = urllib.urlopen(s['safesync_url'])
  1246. rev = handle.read().strip()
  1247. handle.close()
  1248. if len(rev):
  1249. options.revisions.append(s['name']+'@'+rev)
  1250. if options.verbose:
  1251. # Print out the .gclient file. This is longer than if we just printed the
  1252. # client dict, but more legible, and it might contain helpful comments.
  1253. print >>options.stdout, client.ConfigContent()
  1254. return client.RunOnDeps('update', args)
  1255. def DoDiff(options, args):
  1256. """Handle the diff subcommand.
  1257. Raises:
  1258. Error: if client isn't configured properly.
  1259. """
  1260. client = options.gclient.LoadCurrentConfig(options)
  1261. if not client:
  1262. raise Error("client not configured; see 'gclient config'")
  1263. if options.verbose:
  1264. # Print out the .gclient file. This is longer than if we just printed the
  1265. # client dict, but more legible, and it might contain helpful comments.
  1266. print >>options.stdout, client.ConfigContent()
  1267. options.verbose = True
  1268. return client.RunOnDeps('diff', args)
  1269. def DoRevert(options, args):
  1270. """Handle the revert subcommand.
  1271. Raises:
  1272. Error: if client isn't configured properly.
  1273. """
  1274. client = options.gclient.LoadCurrentConfig(options)
  1275. if not client:
  1276. raise Error("client not configured; see 'gclient config'")
  1277. return client.RunOnDeps('revert', args)
  1278. def DoRunHooks(options, args):
  1279. """Handle the runhooks subcommand.
  1280. Raises:
  1281. Error: if client isn't configured properly.
  1282. """
  1283. client = options.gclient.LoadCurrentConfig(options)
  1284. if not client:
  1285. raise Error("client not configured; see 'gclient config'")
  1286. if options.verbose:
  1287. # Print out the .gclient file. This is longer than if we just printed the
  1288. # client dict, but more legible, and it might contain helpful comments.
  1289. print >>options.stdout, client.ConfigContent()
  1290. return client.RunOnDeps('runhooks', args)
  1291. def DoRevInfo(options, args):
  1292. """Handle the revinfo subcommand.
  1293. Raises:
  1294. Error: if client isn't configured properly.
  1295. """
  1296. client = options.gclient.LoadCurrentConfig(options)
  1297. if not client:
  1298. raise Error("client not configured; see 'gclient config'")
  1299. client.PrintRevInfo()
  1300. gclient_command_map = {
  1301. "cleanup": DoCleanup,
  1302. "config": DoConfig,
  1303. "diff": DoDiff,
  1304. "help": DoHelp,
  1305. "status": DoStatus,
  1306. "sync": DoUpdate,
  1307. "update": DoUpdate,
  1308. "revert": DoRevert,
  1309. "runhooks": DoRunHooks,
  1310. "revinfo" : DoRevInfo,
  1311. }
  1312. def DispatchCommand(command, options, args, command_map=None):
  1313. """Dispatches the appropriate subcommand based on command line arguments."""
  1314. if command_map is None:
  1315. command_map = gclient_command_map
  1316. if command in command_map:
  1317. return command_map[command](options, args)
  1318. else:
  1319. raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
  1320. def Main(argv):
  1321. """Parse command line arguments and dispatch command."""
  1322. option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
  1323. version=__version__)
  1324. option_parser.disable_interspersed_args()
  1325. option_parser.add_option("", "--force", action="store_true", default=False,
  1326. help=("(update/sync only) force update even "
  1327. "for modules which haven't changed"))
  1328. option_parser.add_option("", "--revision", action="append", dest="revisions",
  1329. metavar="REV", default=[],
  1330. help=("(update/sync only) sync to a specific "
  1331. "revision, can be used multiple times for "
  1332. "each solution, e.g. --revision=src@123, "
  1333. "--revision=internal@32"))
  1334. option_parser.add_option("", "--deps", default=None, dest="deps_os",
  1335. metavar="OS_LIST",
  1336. help=("(update/sync only) sync deps for the "
  1337. "specified (comma-separated) platform(s); "
  1338. "'all' will sync all platforms"))
  1339. option_parser.add_option("", "--spec", default=None,
  1340. help=("(config only) create a gclient file "
  1341. "containing the provided string"))
  1342. option_parser.add_option("", "--verbose", action="store_true", default=False,
  1343. help="produce additional output for diagnostics")
  1344. option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
  1345. default=False,
  1346. help="Skip svn up whenever possible by requesting "
  1347. "actual HEAD revision from the repository")
  1348. option_parser.add_option("", "--head", action="store_true", default=False,
  1349. help=("skips any safesync_urls specified in "
  1350. "configured solutions"))
  1351. if len(argv) < 2:
  1352. # Users don't need to be told to use the 'help' command.
  1353. option_parser.print_help()
  1354. return 1
  1355. # Add manual support for --version as first argument.
  1356. if argv[1] == '--version':
  1357. option_parser.print_version()
  1358. return 0
  1359. # Add manual support for --help as first argument.
  1360. if argv[1] == '--help':
  1361. argv[1] = 'help'
  1362. command = argv[1]
  1363. options, args = option_parser.parse_args(argv[2:])
  1364. if len(argv) < 3 and command == "help":
  1365. option_parser.print_help()
  1366. return 0
  1367. # Files used for configuration and state saving.
  1368. options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
  1369. options.entries_filename = ".gclient_entries"
  1370. options.deps_file = "DEPS"
  1371. # These are overridded when testing. They are not externally visible.
  1372. options.stdout = sys.stdout
  1373. options.path_exists = os.path.exists
  1374. options.gclient = GClient
  1375. options.scm_wrapper = SCMWrapper
  1376. options.platform = sys.platform
  1377. return DispatchCommand(command, options, args)
  1378. if "__main__" == __name__:
  1379. try:
  1380. result = Main(sys.argv)
  1381. except Error, e:
  1382. print "Error: %s" % str(e)
  1383. result = 1
  1384. sys.exit(result)
  1385. # vim: ts=2:sw=2:tw=80:et: