113481Sgiacomo.travaglini@arm.com#!/usr/bin/env python 213481Sgiacomo.travaglini@arm.com# 313481Sgiacomo.travaglini@arm.com# Copyright 2007 Google Inc. 413481Sgiacomo.travaglini@arm.com# 513481Sgiacomo.travaglini@arm.com# Licensed under the Apache License, Version 2.0 (the "License"); 613481Sgiacomo.travaglini@arm.com# you may not use this file except in compliance with the License. 713481Sgiacomo.travaglini@arm.com# You may obtain a copy of the License at 813481Sgiacomo.travaglini@arm.com# 913481Sgiacomo.travaglini@arm.com# http://www.apache.org/licenses/LICENSE-2.0 1013481Sgiacomo.travaglini@arm.com# 1113481Sgiacomo.travaglini@arm.com# Unless required by applicable law or agreed to in writing, software 1213481Sgiacomo.travaglini@arm.com# distributed under the License is distributed on an "AS IS" BASIS, 1313481Sgiacomo.travaglini@arm.com# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1413481Sgiacomo.travaglini@arm.com# See the License for the specific language governing permissions and 1513481Sgiacomo.travaglini@arm.com# limitations under the License. 1613481Sgiacomo.travaglini@arm.com 1713481Sgiacomo.travaglini@arm.com"""Tool for uploading diffs from a version control system to the codereview app. 1813481Sgiacomo.travaglini@arm.com 1913481Sgiacomo.travaglini@arm.comUsage summary: upload.py [options] [-- diff_options] 2013481Sgiacomo.travaglini@arm.com 2113481Sgiacomo.travaglini@arm.comDiff options are passed to the diff command of the underlying system. 2213481Sgiacomo.travaglini@arm.com 2313481Sgiacomo.travaglini@arm.comSupported version control systems: 2413481Sgiacomo.travaglini@arm.com Git 2513481Sgiacomo.travaglini@arm.com Mercurial 2613481Sgiacomo.travaglini@arm.com Subversion 2713481Sgiacomo.travaglini@arm.com 2813481Sgiacomo.travaglini@arm.comIt is important for Git/Mercurial users to specify a tree/node/branch to diff 2913481Sgiacomo.travaglini@arm.comagainst by using the '--rev' option. 3013481Sgiacomo.travaglini@arm.com""" 3113481Sgiacomo.travaglini@arm.com# This code is derived from appcfg.py in the App Engine SDK (open source), 3213481Sgiacomo.travaglini@arm.com# and from ASPN recipe #146306. 3313481Sgiacomo.travaglini@arm.com 3413481Sgiacomo.travaglini@arm.comimport cookielib 3513481Sgiacomo.travaglini@arm.comimport getpass 3613481Sgiacomo.travaglini@arm.comimport logging 3713481Sgiacomo.travaglini@arm.comimport md5 3813481Sgiacomo.travaglini@arm.comimport mimetypes 3913481Sgiacomo.travaglini@arm.comimport optparse 4013481Sgiacomo.travaglini@arm.comimport os 4113481Sgiacomo.travaglini@arm.comimport re 4213481Sgiacomo.travaglini@arm.comimport socket 4313481Sgiacomo.travaglini@arm.comimport subprocess 4413481Sgiacomo.travaglini@arm.comimport sys 4513481Sgiacomo.travaglini@arm.comimport urllib 4613481Sgiacomo.travaglini@arm.comimport urllib2 4713481Sgiacomo.travaglini@arm.comimport urlparse 4813481Sgiacomo.travaglini@arm.com 4913481Sgiacomo.travaglini@arm.comtry: 5013481Sgiacomo.travaglini@arm.com import readline 5113481Sgiacomo.travaglini@arm.comexcept ImportError: 5213481Sgiacomo.travaglini@arm.com pass 5313481Sgiacomo.travaglini@arm.com 5413481Sgiacomo.travaglini@arm.com# The logging verbosity: 5513481Sgiacomo.travaglini@arm.com# 0: Errors only. 5613481Sgiacomo.travaglini@arm.com# 1: Status messages. 5713481Sgiacomo.travaglini@arm.com# 2: Info logs. 5813481Sgiacomo.travaglini@arm.com# 3: Debug logs. 5913481Sgiacomo.travaglini@arm.comverbosity = 1 6013481Sgiacomo.travaglini@arm.com 6113481Sgiacomo.travaglini@arm.com# Max size of patch or base file. 6213481Sgiacomo.travaglini@arm.comMAX_UPLOAD_SIZE = 900 * 1024 6313481Sgiacomo.travaglini@arm.com 6413481Sgiacomo.travaglini@arm.com 6513481Sgiacomo.travaglini@arm.comdef GetEmail(prompt): 6613481Sgiacomo.travaglini@arm.com """Prompts the user for their email address and returns it. 6713481Sgiacomo.travaglini@arm.com 6813481Sgiacomo.travaglini@arm.com The last used email address is saved to a file and offered up as a suggestion 6913481Sgiacomo.travaglini@arm.com to the user. If the user presses enter without typing in anything the last 7013481Sgiacomo.travaglini@arm.com used email address is used. If the user enters a new address, it is saved 7113481Sgiacomo.travaglini@arm.com for next time we prompt. 7213481Sgiacomo.travaglini@arm.com 7313481Sgiacomo.travaglini@arm.com """ 7413481Sgiacomo.travaglini@arm.com last_email_file_name = os.path.expanduser("~/.last_codereview_email_address") 7513481Sgiacomo.travaglini@arm.com last_email = "" 7613481Sgiacomo.travaglini@arm.com if os.path.exists(last_email_file_name): 7713481Sgiacomo.travaglini@arm.com try: 7813481Sgiacomo.travaglini@arm.com last_email_file = open(last_email_file_name, "r") 7913481Sgiacomo.travaglini@arm.com last_email = last_email_file.readline().strip("\n") 8013481Sgiacomo.travaglini@arm.com last_email_file.close() 8113481Sgiacomo.travaglini@arm.com prompt += " [%s]" % last_email 8213481Sgiacomo.travaglini@arm.com except IOError, e: 8313481Sgiacomo.travaglini@arm.com pass 8413481Sgiacomo.travaglini@arm.com email = raw_input(prompt + ": ").strip() 8513481Sgiacomo.travaglini@arm.com if email: 8613481Sgiacomo.travaglini@arm.com try: 8713481Sgiacomo.travaglini@arm.com last_email_file = open(last_email_file_name, "w") 8813481Sgiacomo.travaglini@arm.com last_email_file.write(email) 8913481Sgiacomo.travaglini@arm.com last_email_file.close() 9013481Sgiacomo.travaglini@arm.com except IOError, e: 9113481Sgiacomo.travaglini@arm.com pass 9213481Sgiacomo.travaglini@arm.com else: 9313481Sgiacomo.travaglini@arm.com email = last_email 9413481Sgiacomo.travaglini@arm.com return email 9513481Sgiacomo.travaglini@arm.com 9613481Sgiacomo.travaglini@arm.com 9713481Sgiacomo.travaglini@arm.comdef StatusUpdate(msg): 9813481Sgiacomo.travaglini@arm.com """Print a status message to stdout. 9913481Sgiacomo.travaglini@arm.com 10013481Sgiacomo.travaglini@arm.com If 'verbosity' is greater than 0, print the message. 10113481Sgiacomo.travaglini@arm.com 10213481Sgiacomo.travaglini@arm.com Args: 10313481Sgiacomo.travaglini@arm.com msg: The string to print. 10413481Sgiacomo.travaglini@arm.com """ 10513481Sgiacomo.travaglini@arm.com if verbosity > 0: 10613481Sgiacomo.travaglini@arm.com print msg 10713481Sgiacomo.travaglini@arm.com 10813481Sgiacomo.travaglini@arm.com 10913481Sgiacomo.travaglini@arm.comdef ErrorExit(msg): 11013481Sgiacomo.travaglini@arm.com """Print an error message to stderr and exit.""" 11113481Sgiacomo.travaglini@arm.com print >>sys.stderr, msg 11213481Sgiacomo.travaglini@arm.com sys.exit(1) 11313481Sgiacomo.travaglini@arm.com 11413481Sgiacomo.travaglini@arm.com 11513481Sgiacomo.travaglini@arm.comclass ClientLoginError(urllib2.HTTPError): 11613481Sgiacomo.travaglini@arm.com """Raised to indicate there was an error authenticating with ClientLogin.""" 11713481Sgiacomo.travaglini@arm.com 11813481Sgiacomo.travaglini@arm.com def __init__(self, url, code, msg, headers, args): 11913481Sgiacomo.travaglini@arm.com urllib2.HTTPError.__init__(self, url, code, msg, headers, None) 12013481Sgiacomo.travaglini@arm.com self.args = args 12113481Sgiacomo.travaglini@arm.com self.reason = args["Error"] 12213481Sgiacomo.travaglini@arm.com 12313481Sgiacomo.travaglini@arm.com 12413481Sgiacomo.travaglini@arm.comclass AbstractRpcServer(object): 12513481Sgiacomo.travaglini@arm.com """Provides a common interface for a simple RPC server.""" 12613481Sgiacomo.travaglini@arm.com 12713481Sgiacomo.travaglini@arm.com def __init__(self, host, auth_function, host_override=None, extra_headers={}, 12813481Sgiacomo.travaglini@arm.com save_cookies=False): 12913481Sgiacomo.travaglini@arm.com """Creates a new HttpRpcServer. 13013481Sgiacomo.travaglini@arm.com 13113481Sgiacomo.travaglini@arm.com Args: 13213481Sgiacomo.travaglini@arm.com host: The host to send requests to. 13313481Sgiacomo.travaglini@arm.com auth_function: A function that takes no arguments and returns an 13413481Sgiacomo.travaglini@arm.com (email, password) tuple when called. Will be called if authentication 13513481Sgiacomo.travaglini@arm.com is required. 13613481Sgiacomo.travaglini@arm.com host_override: The host header to send to the server (defaults to host). 13713481Sgiacomo.travaglini@arm.com extra_headers: A dict of extra headers to append to every request. 13813481Sgiacomo.travaglini@arm.com save_cookies: If True, save the authentication cookies to local disk. 13913481Sgiacomo.travaglini@arm.com If False, use an in-memory cookiejar instead. Subclasses must 14013481Sgiacomo.travaglini@arm.com implement this functionality. Defaults to False. 14113481Sgiacomo.travaglini@arm.com """ 14213481Sgiacomo.travaglini@arm.com self.host = host 14313481Sgiacomo.travaglini@arm.com self.host_override = host_override 14413481Sgiacomo.travaglini@arm.com self.auth_function = auth_function 14513481Sgiacomo.travaglini@arm.com self.authenticated = False 14613481Sgiacomo.travaglini@arm.com self.extra_headers = extra_headers 14713481Sgiacomo.travaglini@arm.com self.save_cookies = save_cookies 14813481Sgiacomo.travaglini@arm.com self.opener = self._GetOpener() 14913481Sgiacomo.travaglini@arm.com if self.host_override: 15013481Sgiacomo.travaglini@arm.com logging.info("Server: %s; Host: %s", self.host, self.host_override) 15113481Sgiacomo.travaglini@arm.com else: 15213481Sgiacomo.travaglini@arm.com logging.info("Server: %s", self.host) 15313481Sgiacomo.travaglini@arm.com 15413481Sgiacomo.travaglini@arm.com def _GetOpener(self): 15513481Sgiacomo.travaglini@arm.com """Returns an OpenerDirector for making HTTP requests. 15613481Sgiacomo.travaglini@arm.com 15713481Sgiacomo.travaglini@arm.com Returns: 15813481Sgiacomo.travaglini@arm.com A urllib2.OpenerDirector object. 15913481Sgiacomo.travaglini@arm.com """ 16013481Sgiacomo.travaglini@arm.com raise NotImplementedError() 16113481Sgiacomo.travaglini@arm.com 16213481Sgiacomo.travaglini@arm.com def _CreateRequest(self, url, data=None): 16313481Sgiacomo.travaglini@arm.com """Creates a new urllib request.""" 16413481Sgiacomo.travaglini@arm.com logging.debug("Creating request for: '%s' with payload:\n%s", url, data) 16513481Sgiacomo.travaglini@arm.com req = urllib2.Request(url, data=data) 16613481Sgiacomo.travaglini@arm.com if self.host_override: 16713481Sgiacomo.travaglini@arm.com req.add_header("Host", self.host_override) 16813481Sgiacomo.travaglini@arm.com for key, value in self.extra_headers.iteritems(): 16913481Sgiacomo.travaglini@arm.com req.add_header(key, value) 17013481Sgiacomo.travaglini@arm.com return req 17113481Sgiacomo.travaglini@arm.com 17213481Sgiacomo.travaglini@arm.com def _GetAuthToken(self, email, password): 17313481Sgiacomo.travaglini@arm.com """Uses ClientLogin to authenticate the user, returning an auth token. 17413481Sgiacomo.travaglini@arm.com 17513481Sgiacomo.travaglini@arm.com Args: 17613481Sgiacomo.travaglini@arm.com email: The user's email address 17713481Sgiacomo.travaglini@arm.com password: The user's password 17813481Sgiacomo.travaglini@arm.com 17913481Sgiacomo.travaglini@arm.com Raises: 18013481Sgiacomo.travaglini@arm.com ClientLoginError: If there was an error authenticating with ClientLogin. 18113481Sgiacomo.travaglini@arm.com HTTPError: If there was some other form of HTTP error. 18213481Sgiacomo.travaglini@arm.com 18313481Sgiacomo.travaglini@arm.com Returns: 18413481Sgiacomo.travaglini@arm.com The authentication token returned by ClientLogin. 18513481Sgiacomo.travaglini@arm.com """ 18613481Sgiacomo.travaglini@arm.com account_type = "GOOGLE" 18713481Sgiacomo.travaglini@arm.com if self.host.endswith(".google.com"): 18813481Sgiacomo.travaglini@arm.com # Needed for use inside Google. 18913481Sgiacomo.travaglini@arm.com account_type = "HOSTED" 19013481Sgiacomo.travaglini@arm.com req = self._CreateRequest( 19113481Sgiacomo.travaglini@arm.com url="https://www.google.com/accounts/ClientLogin", 19213481Sgiacomo.travaglini@arm.com data=urllib.urlencode({ 19313481Sgiacomo.travaglini@arm.com "Email": email, 19413481Sgiacomo.travaglini@arm.com "Passwd": password, 19513481Sgiacomo.travaglini@arm.com "service": "ah", 19613481Sgiacomo.travaglini@arm.com "source": "rietveld-codereview-upload", 19713481Sgiacomo.travaglini@arm.com "accountType": account_type, 19813481Sgiacomo.travaglini@arm.com }), 19913481Sgiacomo.travaglini@arm.com ) 20013481Sgiacomo.travaglini@arm.com try: 20113481Sgiacomo.travaglini@arm.com response = self.opener.open(req) 20213481Sgiacomo.travaglini@arm.com response_body = response.read() 20313481Sgiacomo.travaglini@arm.com response_dict = dict(x.split("=") 20413481Sgiacomo.travaglini@arm.com for x in response_body.split("\n") if x) 20513481Sgiacomo.travaglini@arm.com return response_dict["Auth"] 20613481Sgiacomo.travaglini@arm.com except urllib2.HTTPError, e: 20713481Sgiacomo.travaglini@arm.com if e.code == 403: 20813481Sgiacomo.travaglini@arm.com body = e.read() 20913481Sgiacomo.travaglini@arm.com response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) 21013481Sgiacomo.travaglini@arm.com raise ClientLoginError(req.get_full_url(), e.code, e.msg, 21113481Sgiacomo.travaglini@arm.com e.headers, response_dict) 21213481Sgiacomo.travaglini@arm.com else: 21313481Sgiacomo.travaglini@arm.com raise 21413481Sgiacomo.travaglini@arm.com 21513481Sgiacomo.travaglini@arm.com def _GetAuthCookie(self, auth_token): 21613481Sgiacomo.travaglini@arm.com """Fetches authentication cookies for an authentication token. 21713481Sgiacomo.travaglini@arm.com 21813481Sgiacomo.travaglini@arm.com Args: 21913481Sgiacomo.travaglini@arm.com auth_token: The authentication token returned by ClientLogin. 22013481Sgiacomo.travaglini@arm.com 22113481Sgiacomo.travaglini@arm.com Raises: 22213481Sgiacomo.travaglini@arm.com HTTPError: If there was an error fetching the authentication cookies. 22313481Sgiacomo.travaglini@arm.com """ 22413481Sgiacomo.travaglini@arm.com # This is a dummy value to allow us to identify when we're successful. 22513481Sgiacomo.travaglini@arm.com continue_location = "http://localhost/" 22613481Sgiacomo.travaglini@arm.com args = {"continue": continue_location, "auth": auth_token} 22713481Sgiacomo.travaglini@arm.com req = self._CreateRequest("http://%s/_ah/login?%s" % 22813481Sgiacomo.travaglini@arm.com (self.host, urllib.urlencode(args))) 22913481Sgiacomo.travaglini@arm.com try: 23013481Sgiacomo.travaglini@arm.com response = self.opener.open(req) 23113481Sgiacomo.travaglini@arm.com except urllib2.HTTPError, e: 23213481Sgiacomo.travaglini@arm.com response = e 23313481Sgiacomo.travaglini@arm.com if (response.code != 302 or 23413481Sgiacomo.travaglini@arm.com response.info()["location"] != continue_location): 23513481Sgiacomo.travaglini@arm.com raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, 23613481Sgiacomo.travaglini@arm.com response.headers, response.fp) 23713481Sgiacomo.travaglini@arm.com self.authenticated = True 23813481Sgiacomo.travaglini@arm.com 23913481Sgiacomo.travaglini@arm.com def _Authenticate(self): 24013481Sgiacomo.travaglini@arm.com """Authenticates the user. 24113481Sgiacomo.travaglini@arm.com 24213481Sgiacomo.travaglini@arm.com The authentication process works as follows: 24313481Sgiacomo.travaglini@arm.com 1) We get a username and password from the user 24413481Sgiacomo.travaglini@arm.com 2) We use ClientLogin to obtain an AUTH token for the user 24513481Sgiacomo.travaglini@arm.com (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). 24613481Sgiacomo.travaglini@arm.com 3) We pass the auth token to /_ah/login on the server to obtain an 24713481Sgiacomo.travaglini@arm.com authentication cookie. If login was successful, it tries to redirect 24813481Sgiacomo.travaglini@arm.com us to the URL we provided. 24913481Sgiacomo.travaglini@arm.com 25013481Sgiacomo.travaglini@arm.com If we attempt to access the upload API without first obtaining an 25113481Sgiacomo.travaglini@arm.com authentication cookie, it returns a 401 response and directs us to 25213481Sgiacomo.travaglini@arm.com authenticate ourselves with ClientLogin. 25313481Sgiacomo.travaglini@arm.com """ 25413481Sgiacomo.travaglini@arm.com for i in range(3): 25513481Sgiacomo.travaglini@arm.com credentials = self.auth_function() 25613481Sgiacomo.travaglini@arm.com try: 25713481Sgiacomo.travaglini@arm.com auth_token = self._GetAuthToken(credentials[0], credentials[1]) 25813481Sgiacomo.travaglini@arm.com except ClientLoginError, e: 25913481Sgiacomo.travaglini@arm.com if e.reason == "BadAuthentication": 26013481Sgiacomo.travaglini@arm.com print >>sys.stderr, "Invalid username or password." 26113481Sgiacomo.travaglini@arm.com continue 26213481Sgiacomo.travaglini@arm.com if e.reason == "CaptchaRequired": 26313481Sgiacomo.travaglini@arm.com print >>sys.stderr, ( 26413481Sgiacomo.travaglini@arm.com "Please go to\n" 26513481Sgiacomo.travaglini@arm.com "https://www.google.com/accounts/DisplayUnlockCaptcha\n" 26613481Sgiacomo.travaglini@arm.com "and verify you are a human. Then try again.") 26713481Sgiacomo.travaglini@arm.com break 26813481Sgiacomo.travaglini@arm.com if e.reason == "NotVerified": 26913481Sgiacomo.travaglini@arm.com print >>sys.stderr, "Account not verified." 27013481Sgiacomo.travaglini@arm.com break 27113481Sgiacomo.travaglini@arm.com if e.reason == "TermsNotAgreed": 27213481Sgiacomo.travaglini@arm.com print >>sys.stderr, "User has not agreed to TOS." 27313481Sgiacomo.travaglini@arm.com break 27413481Sgiacomo.travaglini@arm.com if e.reason == "AccountDeleted": 27513481Sgiacomo.travaglini@arm.com print >>sys.stderr, "The user account has been deleted." 27613481Sgiacomo.travaglini@arm.com break 27713481Sgiacomo.travaglini@arm.com if e.reason == "AccountDisabled": 27813481Sgiacomo.travaglini@arm.com print >>sys.stderr, "The user account has been disabled." 27913481Sgiacomo.travaglini@arm.com break 28013481Sgiacomo.travaglini@arm.com if e.reason == "ServiceDisabled": 28113481Sgiacomo.travaglini@arm.com print >>sys.stderr, ("The user's access to the service has been " 28213481Sgiacomo.travaglini@arm.com "disabled.") 28313481Sgiacomo.travaglini@arm.com break 28413481Sgiacomo.travaglini@arm.com if e.reason == "ServiceUnavailable": 28513481Sgiacomo.travaglini@arm.com print >>sys.stderr, "The service is not available; try again later." 28613481Sgiacomo.travaglini@arm.com break 28713481Sgiacomo.travaglini@arm.com raise 28813481Sgiacomo.travaglini@arm.com self._GetAuthCookie(auth_token) 28913481Sgiacomo.travaglini@arm.com return 29013481Sgiacomo.travaglini@arm.com 29113481Sgiacomo.travaglini@arm.com def Send(self, request_path, payload=None, 29213481Sgiacomo.travaglini@arm.com content_type="application/octet-stream", 29313481Sgiacomo.travaglini@arm.com timeout=None, 29413481Sgiacomo.travaglini@arm.com **kwargs): 29513481Sgiacomo.travaglini@arm.com """Sends an RPC and returns the response. 29613481Sgiacomo.travaglini@arm.com 29713481Sgiacomo.travaglini@arm.com Args: 29813481Sgiacomo.travaglini@arm.com request_path: The path to send the request to, eg /api/appversion/create. 29913481Sgiacomo.travaglini@arm.com payload: The body of the request, or None to send an empty request. 30013481Sgiacomo.travaglini@arm.com content_type: The Content-Type header to use. 30113481Sgiacomo.travaglini@arm.com timeout: timeout in seconds; default None i.e. no timeout. 30213481Sgiacomo.travaglini@arm.com (Note: for large requests on OS X, the timeout doesn't work right.) 30313481Sgiacomo.travaglini@arm.com kwargs: Any keyword arguments are converted into query string parameters. 30413481Sgiacomo.travaglini@arm.com 30513481Sgiacomo.travaglini@arm.com Returns: 30613481Sgiacomo.travaglini@arm.com The response body, as a string. 30713481Sgiacomo.travaglini@arm.com """ 30813481Sgiacomo.travaglini@arm.com # TODO: Don't require authentication. Let the server say 30913481Sgiacomo.travaglini@arm.com # whether it is necessary. 31013481Sgiacomo.travaglini@arm.com if not self.authenticated: 31113481Sgiacomo.travaglini@arm.com self._Authenticate() 31213481Sgiacomo.travaglini@arm.com 31313481Sgiacomo.travaglini@arm.com old_timeout = socket.getdefaulttimeout() 31413481Sgiacomo.travaglini@arm.com socket.setdefaulttimeout(timeout) 31513481Sgiacomo.travaglini@arm.com try: 31613481Sgiacomo.travaglini@arm.com tries = 0 31713481Sgiacomo.travaglini@arm.com while True: 31813481Sgiacomo.travaglini@arm.com tries += 1 31913481Sgiacomo.travaglini@arm.com args = dict(kwargs) 32013481Sgiacomo.travaglini@arm.com url = "http://%s%s" % (self.host, request_path) 32113481Sgiacomo.travaglini@arm.com if args: 32213481Sgiacomo.travaglini@arm.com url += "?" + urllib.urlencode(args) 32313481Sgiacomo.travaglini@arm.com req = self._CreateRequest(url=url, data=payload) 32413481Sgiacomo.travaglini@arm.com req.add_header("Content-Type", content_type) 32513481Sgiacomo.travaglini@arm.com try: 32613481Sgiacomo.travaglini@arm.com f = self.opener.open(req) 32713481Sgiacomo.travaglini@arm.com response = f.read() 32813481Sgiacomo.travaglini@arm.com f.close() 32913481Sgiacomo.travaglini@arm.com return response 33013481Sgiacomo.travaglini@arm.com except urllib2.HTTPError, e: 33113481Sgiacomo.travaglini@arm.com if tries > 3: 33213481Sgiacomo.travaglini@arm.com raise 33313481Sgiacomo.travaglini@arm.com elif e.code == 401: 33413481Sgiacomo.travaglini@arm.com self._Authenticate() 33513481Sgiacomo.travaglini@arm.com## elif e.code >= 500 and e.code < 600: 33613481Sgiacomo.travaglini@arm.com## # Server Error - try again. 33713481Sgiacomo.travaglini@arm.com## continue 33813481Sgiacomo.travaglini@arm.com else: 33913481Sgiacomo.travaglini@arm.com raise 34013481Sgiacomo.travaglini@arm.com finally: 34113481Sgiacomo.travaglini@arm.com socket.setdefaulttimeout(old_timeout) 34213481Sgiacomo.travaglini@arm.com 34313481Sgiacomo.travaglini@arm.com 34413481Sgiacomo.travaglini@arm.comclass HttpRpcServer(AbstractRpcServer): 34513481Sgiacomo.travaglini@arm.com """Provides a simplified RPC-style interface for HTTP requests.""" 34613481Sgiacomo.travaglini@arm.com 34713481Sgiacomo.travaglini@arm.com def _Authenticate(self): 34813481Sgiacomo.travaglini@arm.com """Save the cookie jar after authentication.""" 34913481Sgiacomo.travaglini@arm.com super(HttpRpcServer, self)._Authenticate() 35013481Sgiacomo.travaglini@arm.com if self.save_cookies: 35113481Sgiacomo.travaglini@arm.com StatusUpdate("Saving authentication cookies to %s" % self.cookie_file) 35213481Sgiacomo.travaglini@arm.com self.cookie_jar.save() 35313481Sgiacomo.travaglini@arm.com 35413481Sgiacomo.travaglini@arm.com def _GetOpener(self): 35513481Sgiacomo.travaglini@arm.com """Returns an OpenerDirector that supports cookies and ignores redirects. 35613481Sgiacomo.travaglini@arm.com 35713481Sgiacomo.travaglini@arm.com Returns: 35813481Sgiacomo.travaglini@arm.com A urllib2.OpenerDirector object. 35913481Sgiacomo.travaglini@arm.com """ 36013481Sgiacomo.travaglini@arm.com opener = urllib2.OpenerDirector() 36113481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.ProxyHandler()) 36213481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.UnknownHandler()) 36313481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.HTTPHandler()) 36413481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.HTTPDefaultErrorHandler()) 36513481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.HTTPSHandler()) 36613481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.HTTPErrorProcessor()) 36713481Sgiacomo.travaglini@arm.com if self.save_cookies: 36813481Sgiacomo.travaglini@arm.com self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies") 36913481Sgiacomo.travaglini@arm.com self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) 37013481Sgiacomo.travaglini@arm.com if os.path.exists(self.cookie_file): 37113481Sgiacomo.travaglini@arm.com try: 37213481Sgiacomo.travaglini@arm.com self.cookie_jar.load() 37313481Sgiacomo.travaglini@arm.com self.authenticated = True 37413481Sgiacomo.travaglini@arm.com StatusUpdate("Loaded authentication cookies from %s" % 37513481Sgiacomo.travaglini@arm.com self.cookie_file) 37613481Sgiacomo.travaglini@arm.com except (cookielib.LoadError, IOError): 37713481Sgiacomo.travaglini@arm.com # Failed to load cookies - just ignore them. 37813481Sgiacomo.travaglini@arm.com pass 37913481Sgiacomo.travaglini@arm.com else: 38013481Sgiacomo.travaglini@arm.com # Create an empty cookie file with mode 600 38113481Sgiacomo.travaglini@arm.com fd = os.open(self.cookie_file, os.O_CREAT, 0600) 38213481Sgiacomo.travaglini@arm.com os.close(fd) 38313481Sgiacomo.travaglini@arm.com # Always chmod the cookie file 38413481Sgiacomo.travaglini@arm.com os.chmod(self.cookie_file, 0600) 38513481Sgiacomo.travaglini@arm.com else: 38613481Sgiacomo.travaglini@arm.com # Don't save cookies across runs of update.py. 38713481Sgiacomo.travaglini@arm.com self.cookie_jar = cookielib.CookieJar() 38813481Sgiacomo.travaglini@arm.com opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) 38913481Sgiacomo.travaglini@arm.com return opener 39013481Sgiacomo.travaglini@arm.com 39113481Sgiacomo.travaglini@arm.com 39213481Sgiacomo.travaglini@arm.comparser = optparse.OptionParser(usage="%prog [options] [-- diff_options]") 39313481Sgiacomo.travaglini@arm.comparser.add_option("-y", "--assume_yes", action="store_true", 39413481Sgiacomo.travaglini@arm.com dest="assume_yes", default=False, 39513481Sgiacomo.travaglini@arm.com help="Assume that the answer to yes/no questions is 'yes'.") 39613481Sgiacomo.travaglini@arm.com# Logging 39713481Sgiacomo.travaglini@arm.comgroup = parser.add_option_group("Logging options") 39813481Sgiacomo.travaglini@arm.comgroup.add_option("-q", "--quiet", action="store_const", const=0, 39913481Sgiacomo.travaglini@arm.com dest="verbose", help="Print errors only.") 40013481Sgiacomo.travaglini@arm.comgroup.add_option("-v", "--verbose", action="store_const", const=2, 40113481Sgiacomo.travaglini@arm.com dest="verbose", default=1, 40213481Sgiacomo.travaglini@arm.com help="Print info level logs (default).") 40313481Sgiacomo.travaglini@arm.comgroup.add_option("--noisy", action="store_const", const=3, 40413481Sgiacomo.travaglini@arm.com dest="verbose", help="Print all logs.") 40513481Sgiacomo.travaglini@arm.com# Review server 40613481Sgiacomo.travaglini@arm.comgroup = parser.add_option_group("Review server options") 40713481Sgiacomo.travaglini@arm.comgroup.add_option("-s", "--server", action="store", dest="server", 40813481Sgiacomo.travaglini@arm.com default="codereview.appspot.com", 40913481Sgiacomo.travaglini@arm.com metavar="SERVER", 41013481Sgiacomo.travaglini@arm.com help=("The server to upload to. The format is host[:port]. " 41113481Sgiacomo.travaglini@arm.com "Defaults to 'codereview.appspot.com'.")) 41213481Sgiacomo.travaglini@arm.comgroup.add_option("-e", "--email", action="store", dest="email", 41313481Sgiacomo.travaglini@arm.com metavar="EMAIL", default=None, 41413481Sgiacomo.travaglini@arm.com help="The username to use. Will prompt if omitted.") 41513481Sgiacomo.travaglini@arm.comgroup.add_option("-H", "--host", action="store", dest="host", 41613481Sgiacomo.travaglini@arm.com metavar="HOST", default=None, 41713481Sgiacomo.travaglini@arm.com help="Overrides the Host header sent with all RPCs.") 41813481Sgiacomo.travaglini@arm.comgroup.add_option("--no_cookies", action="store_false", 41913481Sgiacomo.travaglini@arm.com dest="save_cookies", default=True, 42013481Sgiacomo.travaglini@arm.com help="Do not save authentication cookies to local disk.") 42113481Sgiacomo.travaglini@arm.com# Issue 42213481Sgiacomo.travaglini@arm.comgroup = parser.add_option_group("Issue options") 42313481Sgiacomo.travaglini@arm.comgroup.add_option("-d", "--description", action="store", dest="description", 42413481Sgiacomo.travaglini@arm.com metavar="DESCRIPTION", default=None, 42513481Sgiacomo.travaglini@arm.com help="Optional description when creating an issue.") 42613481Sgiacomo.travaglini@arm.comgroup.add_option("-f", "--description_file", action="store", 42713481Sgiacomo.travaglini@arm.com dest="description_file", metavar="DESCRIPTION_FILE", 42813481Sgiacomo.travaglini@arm.com default=None, 42913481Sgiacomo.travaglini@arm.com help="Optional path of a file that contains " 43013481Sgiacomo.travaglini@arm.com "the description when creating an issue.") 43113481Sgiacomo.travaglini@arm.comgroup.add_option("-r", "--reviewers", action="store", dest="reviewers", 43213481Sgiacomo.travaglini@arm.com metavar="REVIEWERS", default=None, 43313481Sgiacomo.travaglini@arm.com help="Add reviewers (comma separated email addresses).") 43413481Sgiacomo.travaglini@arm.comgroup.add_option("--cc", action="store", dest="cc", 43513481Sgiacomo.travaglini@arm.com metavar="CC", default=None, 43613481Sgiacomo.travaglini@arm.com help="Add CC (comma separated email addresses).") 43713481Sgiacomo.travaglini@arm.com# Upload options 43813481Sgiacomo.travaglini@arm.comgroup = parser.add_option_group("Patch options") 43913481Sgiacomo.travaglini@arm.comgroup.add_option("-m", "--message", action="store", dest="message", 44013481Sgiacomo.travaglini@arm.com metavar="MESSAGE", default=None, 44113481Sgiacomo.travaglini@arm.com help="A message to identify the patch. " 44213481Sgiacomo.travaglini@arm.com "Will prompt if omitted.") 44313481Sgiacomo.travaglini@arm.comgroup.add_option("-i", "--issue", type="int", action="store", 44413481Sgiacomo.travaglini@arm.com metavar="ISSUE", default=None, 44513481Sgiacomo.travaglini@arm.com help="Issue number to which to add. Defaults to new issue.") 44613481Sgiacomo.travaglini@arm.comgroup.add_option("--download_base", action="store_true", 44713481Sgiacomo.travaglini@arm.com dest="download_base", default=False, 44813481Sgiacomo.travaglini@arm.com help="Base files will be downloaded by the server " 44913481Sgiacomo.travaglini@arm.com "(side-by-side diffs may not work on files with CRs).") 45013481Sgiacomo.travaglini@arm.comgroup.add_option("--rev", action="store", dest="revision", 45113481Sgiacomo.travaglini@arm.com metavar="REV", default=None, 45213481Sgiacomo.travaglini@arm.com help="Branch/tree/revision to diff against (used by DVCS).") 45313481Sgiacomo.travaglini@arm.comgroup.add_option("--send_mail", action="store_true", 45413481Sgiacomo.travaglini@arm.com dest="send_mail", default=False, 45513481Sgiacomo.travaglini@arm.com help="Send notification email to reviewers.") 45613481Sgiacomo.travaglini@arm.com 45713481Sgiacomo.travaglini@arm.com 45813481Sgiacomo.travaglini@arm.comdef GetRpcServer(options): 45913481Sgiacomo.travaglini@arm.com """Returns an instance of an AbstractRpcServer. 46013481Sgiacomo.travaglini@arm.com 46113481Sgiacomo.travaglini@arm.com Returns: 46213481Sgiacomo.travaglini@arm.com A new AbstractRpcServer, on which RPC calls can be made. 46313481Sgiacomo.travaglini@arm.com """ 46413481Sgiacomo.travaglini@arm.com 46513481Sgiacomo.travaglini@arm.com rpc_server_class = HttpRpcServer 46613481Sgiacomo.travaglini@arm.com 46713481Sgiacomo.travaglini@arm.com def GetUserCredentials(): 46813481Sgiacomo.travaglini@arm.com """Prompts the user for a username and password.""" 46913481Sgiacomo.travaglini@arm.com email = options.email 47013481Sgiacomo.travaglini@arm.com if email is None: 47113481Sgiacomo.travaglini@arm.com email = GetEmail("Email (login for uploading to %s)" % options.server) 47213481Sgiacomo.travaglini@arm.com password = getpass.getpass("Password for %s: " % email) 47313481Sgiacomo.travaglini@arm.com return (email, password) 47413481Sgiacomo.travaglini@arm.com 47513481Sgiacomo.travaglini@arm.com # If this is the dev_appserver, use fake authentication. 47613481Sgiacomo.travaglini@arm.com host = (options.host or options.server).lower() 47713481Sgiacomo.travaglini@arm.com if host == "localhost" or host.startswith("localhost:"): 47813481Sgiacomo.travaglini@arm.com email = options.email 47913481Sgiacomo.travaglini@arm.com if email is None: 48013481Sgiacomo.travaglini@arm.com email = "test@example.com" 48113481Sgiacomo.travaglini@arm.com logging.info("Using debug user %s. Override with --email" % email) 48213481Sgiacomo.travaglini@arm.com server = rpc_server_class( 48313481Sgiacomo.travaglini@arm.com options.server, 48413481Sgiacomo.travaglini@arm.com lambda: (email, "password"), 48513481Sgiacomo.travaglini@arm.com host_override=options.host, 48613481Sgiacomo.travaglini@arm.com extra_headers={"Cookie": 48713481Sgiacomo.travaglini@arm.com 'dev_appserver_login="%s:False"' % email}, 48813481Sgiacomo.travaglini@arm.com save_cookies=options.save_cookies) 48913481Sgiacomo.travaglini@arm.com # Don't try to talk to ClientLogin. 49013481Sgiacomo.travaglini@arm.com server.authenticated = True 49113481Sgiacomo.travaglini@arm.com return server 49213481Sgiacomo.travaglini@arm.com 49313481Sgiacomo.travaglini@arm.com return rpc_server_class(options.server, GetUserCredentials, 49413481Sgiacomo.travaglini@arm.com host_override=options.host, 49513481Sgiacomo.travaglini@arm.com save_cookies=options.save_cookies) 49613481Sgiacomo.travaglini@arm.com 49713481Sgiacomo.travaglini@arm.com 49813481Sgiacomo.travaglini@arm.comdef EncodeMultipartFormData(fields, files): 49913481Sgiacomo.travaglini@arm.com """Encode form fields for multipart/form-data. 50013481Sgiacomo.travaglini@arm.com 50113481Sgiacomo.travaglini@arm.com Args: 50213481Sgiacomo.travaglini@arm.com fields: A sequence of (name, value) elements for regular form fields. 50313481Sgiacomo.travaglini@arm.com files: A sequence of (name, filename, value) elements for data to be 50413481Sgiacomo.travaglini@arm.com uploaded as files. 50513481Sgiacomo.travaglini@arm.com Returns: 50613481Sgiacomo.travaglini@arm.com (content_type, body) ready for httplib.HTTP instance. 50713481Sgiacomo.travaglini@arm.com 50813481Sgiacomo.travaglini@arm.com Source: 50913481Sgiacomo.travaglini@arm.com http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 51013481Sgiacomo.travaglini@arm.com """ 51113481Sgiacomo.travaglini@arm.com BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' 51213481Sgiacomo.travaglini@arm.com CRLF = '\r\n' 51313481Sgiacomo.travaglini@arm.com lines = [] 51413481Sgiacomo.travaglini@arm.com for (key, value) in fields: 51513481Sgiacomo.travaglini@arm.com lines.append('--' + BOUNDARY) 51613481Sgiacomo.travaglini@arm.com lines.append('Content-Disposition: form-data; name="%s"' % key) 51713481Sgiacomo.travaglini@arm.com lines.append('') 51813481Sgiacomo.travaglini@arm.com lines.append(value) 51913481Sgiacomo.travaglini@arm.com for (key, filename, value) in files: 52013481Sgiacomo.travaglini@arm.com lines.append('--' + BOUNDARY) 52113481Sgiacomo.travaglini@arm.com lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % 52213481Sgiacomo.travaglini@arm.com (key, filename)) 52313481Sgiacomo.travaglini@arm.com lines.append('Content-Type: %s' % GetContentType(filename)) 52413481Sgiacomo.travaglini@arm.com lines.append('') 52513481Sgiacomo.travaglini@arm.com lines.append(value) 52613481Sgiacomo.travaglini@arm.com lines.append('--' + BOUNDARY + '--') 52713481Sgiacomo.travaglini@arm.com lines.append('') 52813481Sgiacomo.travaglini@arm.com body = CRLF.join(lines) 52913481Sgiacomo.travaglini@arm.com content_type = 'multipart/form-data; boundary=%s' % BOUNDARY 53013481Sgiacomo.travaglini@arm.com return content_type, body 53113481Sgiacomo.travaglini@arm.com 53213481Sgiacomo.travaglini@arm.com 53313481Sgiacomo.travaglini@arm.comdef GetContentType(filename): 53413481Sgiacomo.travaglini@arm.com """Helper to guess the content-type from the filename.""" 53513481Sgiacomo.travaglini@arm.com return mimetypes.guess_type(filename)[0] or 'application/octet-stream' 53613481Sgiacomo.travaglini@arm.com 53713481Sgiacomo.travaglini@arm.com 53813481Sgiacomo.travaglini@arm.com# Use a shell for subcommands on Windows to get a PATH search. 53913481Sgiacomo.travaglini@arm.comuse_shell = sys.platform.startswith("win") 54013481Sgiacomo.travaglini@arm.com 54113481Sgiacomo.travaglini@arm.comdef RunShellWithReturnCode(command, print_output=False, 54213481Sgiacomo.travaglini@arm.com universal_newlines=True): 54313481Sgiacomo.travaglini@arm.com """Executes a command and returns the output from stdout and the return code. 54413481Sgiacomo.travaglini@arm.com 54513481Sgiacomo.travaglini@arm.com Args: 54613481Sgiacomo.travaglini@arm.com command: Command to execute. 54713481Sgiacomo.travaglini@arm.com print_output: If True, the output is printed to stdout. 54813481Sgiacomo.travaglini@arm.com If False, both stdout and stderr are ignored. 54913481Sgiacomo.travaglini@arm.com universal_newlines: Use universal_newlines flag (default: True). 55013481Sgiacomo.travaglini@arm.com 55113481Sgiacomo.travaglini@arm.com Returns: 55213481Sgiacomo.travaglini@arm.com Tuple (output, return code) 55313481Sgiacomo.travaglini@arm.com """ 55413481Sgiacomo.travaglini@arm.com logging.info("Running %s", command) 55513481Sgiacomo.travaglini@arm.com p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 55613481Sgiacomo.travaglini@arm.com shell=use_shell, universal_newlines=universal_newlines) 55713481Sgiacomo.travaglini@arm.com if print_output: 55813481Sgiacomo.travaglini@arm.com output_array = [] 55913481Sgiacomo.travaglini@arm.com while True: 56013481Sgiacomo.travaglini@arm.com line = p.stdout.readline() 56113481Sgiacomo.travaglini@arm.com if not line: 56213481Sgiacomo.travaglini@arm.com break 56313481Sgiacomo.travaglini@arm.com print line.strip("\n") 56413481Sgiacomo.travaglini@arm.com output_array.append(line) 56513481Sgiacomo.travaglini@arm.com output = "".join(output_array) 56613481Sgiacomo.travaglini@arm.com else: 56713481Sgiacomo.travaglini@arm.com output = p.stdout.read() 56813481Sgiacomo.travaglini@arm.com p.wait() 56913481Sgiacomo.travaglini@arm.com errout = p.stderr.read() 57013481Sgiacomo.travaglini@arm.com if print_output and errout: 57113481Sgiacomo.travaglini@arm.com print >>sys.stderr, errout 57213481Sgiacomo.travaglini@arm.com p.stdout.close() 57313481Sgiacomo.travaglini@arm.com p.stderr.close() 57413481Sgiacomo.travaglini@arm.com return output, p.returncode 57513481Sgiacomo.travaglini@arm.com 57613481Sgiacomo.travaglini@arm.com 57713481Sgiacomo.travaglini@arm.comdef RunShell(command, silent_ok=False, universal_newlines=True, 57813481Sgiacomo.travaglini@arm.com print_output=False): 57913481Sgiacomo.travaglini@arm.com data, retcode = RunShellWithReturnCode(command, print_output, 58013481Sgiacomo.travaglini@arm.com universal_newlines) 58113481Sgiacomo.travaglini@arm.com if retcode: 58213481Sgiacomo.travaglini@arm.com ErrorExit("Got error status from %s:\n%s" % (command, data)) 58313481Sgiacomo.travaglini@arm.com if not silent_ok and not data: 58413481Sgiacomo.travaglini@arm.com ErrorExit("No output from %s" % command) 58513481Sgiacomo.travaglini@arm.com return data 58613481Sgiacomo.travaglini@arm.com 58713481Sgiacomo.travaglini@arm.com 58813481Sgiacomo.travaglini@arm.comclass VersionControlSystem(object): 58913481Sgiacomo.travaglini@arm.com """Abstract base class providing an interface to the VCS.""" 59013481Sgiacomo.travaglini@arm.com 59113481Sgiacomo.travaglini@arm.com def __init__(self, options): 59213481Sgiacomo.travaglini@arm.com """Constructor. 59313481Sgiacomo.travaglini@arm.com 59413481Sgiacomo.travaglini@arm.com Args: 59513481Sgiacomo.travaglini@arm.com options: Command line options. 59613481Sgiacomo.travaglini@arm.com """ 59713481Sgiacomo.travaglini@arm.com self.options = options 59813481Sgiacomo.travaglini@arm.com 59913481Sgiacomo.travaglini@arm.com def GenerateDiff(self, args): 60013481Sgiacomo.travaglini@arm.com """Return the current diff as a string. 60113481Sgiacomo.travaglini@arm.com 60213481Sgiacomo.travaglini@arm.com Args: 60313481Sgiacomo.travaglini@arm.com args: Extra arguments to pass to the diff command. 60413481Sgiacomo.travaglini@arm.com """ 60513481Sgiacomo.travaglini@arm.com raise NotImplementedError( 60613481Sgiacomo.travaglini@arm.com "abstract method -- subclass %s must override" % self.__class__) 60713481Sgiacomo.travaglini@arm.com 60813481Sgiacomo.travaglini@arm.com def GetUnknownFiles(self): 60913481Sgiacomo.travaglini@arm.com """Return a list of files unknown to the VCS.""" 61013481Sgiacomo.travaglini@arm.com raise NotImplementedError( 61113481Sgiacomo.travaglini@arm.com "abstract method -- subclass %s must override" % self.__class__) 61213481Sgiacomo.travaglini@arm.com 61313481Sgiacomo.travaglini@arm.com def CheckForUnknownFiles(self): 61413481Sgiacomo.travaglini@arm.com """Show an "are you sure?" prompt if there are unknown files.""" 61513481Sgiacomo.travaglini@arm.com unknown_files = self.GetUnknownFiles() 61613481Sgiacomo.travaglini@arm.com if unknown_files: 61713481Sgiacomo.travaglini@arm.com print "The following files are not added to version control:" 61813481Sgiacomo.travaglini@arm.com for line in unknown_files: 61913481Sgiacomo.travaglini@arm.com print line 62013481Sgiacomo.travaglini@arm.com prompt = "Are you sure to continue?(y/N) " 62113481Sgiacomo.travaglini@arm.com answer = raw_input(prompt).strip() 62213481Sgiacomo.travaglini@arm.com if answer != "y": 62313481Sgiacomo.travaglini@arm.com ErrorExit("User aborted") 62413481Sgiacomo.travaglini@arm.com 62513481Sgiacomo.travaglini@arm.com def GetBaseFile(self, filename): 62613481Sgiacomo.travaglini@arm.com """Get the content of the upstream version of a file. 62713481Sgiacomo.travaglini@arm.com 62813481Sgiacomo.travaglini@arm.com Returns: 62913481Sgiacomo.travaglini@arm.com A tuple (base_content, new_content, is_binary, status) 63013481Sgiacomo.travaglini@arm.com base_content: The contents of the base file. 63113481Sgiacomo.travaglini@arm.com new_content: For text files, this is empty. For binary files, this is 63213481Sgiacomo.travaglini@arm.com the contents of the new file, since the diff output won't contain 63313481Sgiacomo.travaglini@arm.com information to reconstruct the current file. 63413481Sgiacomo.travaglini@arm.com is_binary: True iff the file is binary. 63513481Sgiacomo.travaglini@arm.com status: The status of the file. 63613481Sgiacomo.travaglini@arm.com """ 63713481Sgiacomo.travaglini@arm.com 63813481Sgiacomo.travaglini@arm.com raise NotImplementedError( 63913481Sgiacomo.travaglini@arm.com "abstract method -- subclass %s must override" % self.__class__) 64013481Sgiacomo.travaglini@arm.com 64113481Sgiacomo.travaglini@arm.com 64213481Sgiacomo.travaglini@arm.com def GetBaseFiles(self, diff): 64313481Sgiacomo.travaglini@arm.com """Helper that calls GetBase file for each file in the patch. 64413481Sgiacomo.travaglini@arm.com 64513481Sgiacomo.travaglini@arm.com Returns: 64613481Sgiacomo.travaglini@arm.com A dictionary that maps from filename to GetBaseFile's tuple. Filenames 64713481Sgiacomo.travaglini@arm.com are retrieved based on lines that start with "Index:" or 64813481Sgiacomo.travaglini@arm.com "Property changes on:". 64913481Sgiacomo.travaglini@arm.com """ 65013481Sgiacomo.travaglini@arm.com files = {} 65113481Sgiacomo.travaglini@arm.com for line in diff.splitlines(True): 65213481Sgiacomo.travaglini@arm.com if line.startswith('Index:') or line.startswith('Property changes on:'): 65313481Sgiacomo.travaglini@arm.com unused, filename = line.split(':', 1) 65413481Sgiacomo.travaglini@arm.com # On Windows if a file has property changes its filename uses '\' 65513481Sgiacomo.travaglini@arm.com # instead of '/'. 65613481Sgiacomo.travaglini@arm.com filename = filename.strip().replace('\\', '/') 65713481Sgiacomo.travaglini@arm.com files[filename] = self.GetBaseFile(filename) 65813481Sgiacomo.travaglini@arm.com return files 65913481Sgiacomo.travaglini@arm.com 66013481Sgiacomo.travaglini@arm.com 66113481Sgiacomo.travaglini@arm.com def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options, 66213481Sgiacomo.travaglini@arm.com files): 66313481Sgiacomo.travaglini@arm.com """Uploads the base files (and if necessary, the current ones as well).""" 66413481Sgiacomo.travaglini@arm.com 66513481Sgiacomo.travaglini@arm.com def UploadFile(filename, file_id, content, is_binary, status, is_base): 66613481Sgiacomo.travaglini@arm.com """Uploads a file to the server.""" 66713481Sgiacomo.travaglini@arm.com file_too_large = False 66813481Sgiacomo.travaglini@arm.com if is_base: 66913481Sgiacomo.travaglini@arm.com type = "base" 67013481Sgiacomo.travaglini@arm.com else: 67113481Sgiacomo.travaglini@arm.com type = "current" 67213481Sgiacomo.travaglini@arm.com if len(content) > MAX_UPLOAD_SIZE: 67313481Sgiacomo.travaglini@arm.com print ("Not uploading the %s file for %s because it's too large." % 67413481Sgiacomo.travaglini@arm.com (type, filename)) 67513481Sgiacomo.travaglini@arm.com file_too_large = True 67613481Sgiacomo.travaglini@arm.com content = "" 67713481Sgiacomo.travaglini@arm.com checksum = md5.new(content).hexdigest() 67813481Sgiacomo.travaglini@arm.com if options.verbose > 0 and not file_too_large: 67913481Sgiacomo.travaglini@arm.com print "Uploading %s file for %s" % (type, filename) 68013481Sgiacomo.travaglini@arm.com url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id) 68113481Sgiacomo.travaglini@arm.com form_fields = [("filename", filename), 68213481Sgiacomo.travaglini@arm.com ("status", status), 68313481Sgiacomo.travaglini@arm.com ("checksum", checksum), 68413481Sgiacomo.travaglini@arm.com ("is_binary", str(is_binary)), 68513481Sgiacomo.travaglini@arm.com ("is_current", str(not is_base)), 68613481Sgiacomo.travaglini@arm.com ] 68713481Sgiacomo.travaglini@arm.com if file_too_large: 68813481Sgiacomo.travaglini@arm.com form_fields.append(("file_too_large", "1")) 68913481Sgiacomo.travaglini@arm.com if options.email: 69013481Sgiacomo.travaglini@arm.com form_fields.append(("user", options.email)) 69113481Sgiacomo.travaglini@arm.com ctype, body = EncodeMultipartFormData(form_fields, 69213481Sgiacomo.travaglini@arm.com [("data", filename, content)]) 69313481Sgiacomo.travaglini@arm.com response_body = rpc_server.Send(url, body, 69413481Sgiacomo.travaglini@arm.com content_type=ctype) 69513481Sgiacomo.travaglini@arm.com if not response_body.startswith("OK"): 69613481Sgiacomo.travaglini@arm.com StatusUpdate(" --> %s" % response_body) 69713481Sgiacomo.travaglini@arm.com sys.exit(1) 69813481Sgiacomo.travaglini@arm.com 69913481Sgiacomo.travaglini@arm.com patches = dict() 70013481Sgiacomo.travaglini@arm.com [patches.setdefault(v, k) for k, v in patch_list] 70113481Sgiacomo.travaglini@arm.com for filename in patches.keys(): 70213481Sgiacomo.travaglini@arm.com base_content, new_content, is_binary, status = files[filename] 70313481Sgiacomo.travaglini@arm.com file_id_str = patches.get(filename) 70413481Sgiacomo.travaglini@arm.com if file_id_str.find("nobase") != -1: 70513481Sgiacomo.travaglini@arm.com base_content = None 70613481Sgiacomo.travaglini@arm.com file_id_str = file_id_str[file_id_str.rfind("_") + 1:] 70713481Sgiacomo.travaglini@arm.com file_id = int(file_id_str) 70813481Sgiacomo.travaglini@arm.com if base_content != None: 70913481Sgiacomo.travaglini@arm.com UploadFile(filename, file_id, base_content, is_binary, status, True) 71013481Sgiacomo.travaglini@arm.com if new_content != None: 71113481Sgiacomo.travaglini@arm.com UploadFile(filename, file_id, new_content, is_binary, status, False) 71213481Sgiacomo.travaglini@arm.com 71313481Sgiacomo.travaglini@arm.com def IsImage(self, filename): 71413481Sgiacomo.travaglini@arm.com """Returns true if the filename has an image extension.""" 71513481Sgiacomo.travaglini@arm.com mimetype = mimetypes.guess_type(filename)[0] 71613481Sgiacomo.travaglini@arm.com if not mimetype: 71713481Sgiacomo.travaglini@arm.com return False 71813481Sgiacomo.travaglini@arm.com return mimetype.startswith("image/") 71913481Sgiacomo.travaglini@arm.com 72013481Sgiacomo.travaglini@arm.com 72113481Sgiacomo.travaglini@arm.comclass SubversionVCS(VersionControlSystem): 72213481Sgiacomo.travaglini@arm.com """Implementation of the VersionControlSystem interface for Subversion.""" 72313481Sgiacomo.travaglini@arm.com 72413481Sgiacomo.travaglini@arm.com def __init__(self, options): 72513481Sgiacomo.travaglini@arm.com super(SubversionVCS, self).__init__(options) 72613481Sgiacomo.travaglini@arm.com if self.options.revision: 72713481Sgiacomo.travaglini@arm.com match = re.match(r"(\d+)(:(\d+))?", self.options.revision) 72813481Sgiacomo.travaglini@arm.com if not match: 72913481Sgiacomo.travaglini@arm.com ErrorExit("Invalid Subversion revision %s." % self.options.revision) 73013481Sgiacomo.travaglini@arm.com self.rev_start = match.group(1) 73113481Sgiacomo.travaglini@arm.com self.rev_end = match.group(3) 73213481Sgiacomo.travaglini@arm.com else: 73313481Sgiacomo.travaglini@arm.com self.rev_start = self.rev_end = None 73413481Sgiacomo.travaglini@arm.com # Cache output from "svn list -r REVNO dirname". 73513481Sgiacomo.travaglini@arm.com # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev). 73613481Sgiacomo.travaglini@arm.com self.svnls_cache = {} 73713481Sgiacomo.travaglini@arm.com # SVN base URL is required to fetch files deleted in an older revision. 73813481Sgiacomo.travaglini@arm.com # Result is cached to not guess it over and over again in GetBaseFile(). 73913481Sgiacomo.travaglini@arm.com required = self.options.download_base or self.options.revision is not None 74013481Sgiacomo.travaglini@arm.com self.svn_base = self._GuessBase(required) 74113481Sgiacomo.travaglini@arm.com 74213481Sgiacomo.travaglini@arm.com def GuessBase(self, required): 74313481Sgiacomo.travaglini@arm.com """Wrapper for _GuessBase.""" 74413481Sgiacomo.travaglini@arm.com return self.svn_base 74513481Sgiacomo.travaglini@arm.com 74613481Sgiacomo.travaglini@arm.com def _GuessBase(self, required): 74713481Sgiacomo.travaglini@arm.com """Returns the SVN base URL. 74813481Sgiacomo.travaglini@arm.com 74913481Sgiacomo.travaglini@arm.com Args: 75013481Sgiacomo.travaglini@arm.com required: If true, exits if the url can't be guessed, otherwise None is 75113481Sgiacomo.travaglini@arm.com returned. 75213481Sgiacomo.travaglini@arm.com """ 75313481Sgiacomo.travaglini@arm.com info = RunShell(["svn", "info"]) 75413481Sgiacomo.travaglini@arm.com for line in info.splitlines(): 75513481Sgiacomo.travaglini@arm.com words = line.split() 75613481Sgiacomo.travaglini@arm.com if len(words) == 2 and words[0] == "URL:": 75713481Sgiacomo.travaglini@arm.com url = words[1] 75813481Sgiacomo.travaglini@arm.com scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) 75913481Sgiacomo.travaglini@arm.com username, netloc = urllib.splituser(netloc) 76013481Sgiacomo.travaglini@arm.com if username: 76113481Sgiacomo.travaglini@arm.com logging.info("Removed username from base URL") 76213481Sgiacomo.travaglini@arm.com if netloc.endswith("svn.python.org"): 76313481Sgiacomo.travaglini@arm.com if netloc == "svn.python.org": 76413481Sgiacomo.travaglini@arm.com if path.startswith("/projects/"): 76513481Sgiacomo.travaglini@arm.com path = path[9:] 76613481Sgiacomo.travaglini@arm.com elif netloc != "pythondev@svn.python.org": 76713481Sgiacomo.travaglini@arm.com ErrorExit("Unrecognized Python URL: %s" % url) 76813481Sgiacomo.travaglini@arm.com base = "http://svn.python.org/view/*checkout*%s/" % path 76913481Sgiacomo.travaglini@arm.com logging.info("Guessed Python base = %s", base) 77013481Sgiacomo.travaglini@arm.com elif netloc.endswith("svn.collab.net"): 77113481Sgiacomo.travaglini@arm.com if path.startswith("/repos/"): 77213481Sgiacomo.travaglini@arm.com path = path[6:] 77313481Sgiacomo.travaglini@arm.com base = "http://svn.collab.net/viewvc/*checkout*%s/" % path 77413481Sgiacomo.travaglini@arm.com logging.info("Guessed CollabNet base = %s", base) 77513481Sgiacomo.travaglini@arm.com elif netloc.endswith(".googlecode.com"): 77613481Sgiacomo.travaglini@arm.com path = path + "/" 77713481Sgiacomo.travaglini@arm.com base = urlparse.urlunparse(("http", netloc, path, params, 77813481Sgiacomo.travaglini@arm.com query, fragment)) 77913481Sgiacomo.travaglini@arm.com logging.info("Guessed Google Code base = %s", base) 78013481Sgiacomo.travaglini@arm.com else: 78113481Sgiacomo.travaglini@arm.com path = path + "/" 78213481Sgiacomo.travaglini@arm.com base = urlparse.urlunparse((scheme, netloc, path, params, 78313481Sgiacomo.travaglini@arm.com query, fragment)) 78413481Sgiacomo.travaglini@arm.com logging.info("Guessed base = %s", base) 78513481Sgiacomo.travaglini@arm.com return base 78613481Sgiacomo.travaglini@arm.com if required: 78713481Sgiacomo.travaglini@arm.com ErrorExit("Can't find URL in output from svn info") 78813481Sgiacomo.travaglini@arm.com return None 78913481Sgiacomo.travaglini@arm.com 79013481Sgiacomo.travaglini@arm.com def GenerateDiff(self, args): 79113481Sgiacomo.travaglini@arm.com cmd = ["svn", "diff"] 79213481Sgiacomo.travaglini@arm.com if self.options.revision: 79313481Sgiacomo.travaglini@arm.com cmd += ["-r", self.options.revision] 79413481Sgiacomo.travaglini@arm.com cmd.extend(args) 79513481Sgiacomo.travaglini@arm.com data = RunShell(cmd) 79613481Sgiacomo.travaglini@arm.com count = 0 79713481Sgiacomo.travaglini@arm.com for line in data.splitlines(): 79813481Sgiacomo.travaglini@arm.com if line.startswith("Index:") or line.startswith("Property changes on:"): 79913481Sgiacomo.travaglini@arm.com count += 1 80013481Sgiacomo.travaglini@arm.com logging.info(line) 80113481Sgiacomo.travaglini@arm.com if not count: 80213481Sgiacomo.travaglini@arm.com ErrorExit("No valid patches found in output from svn diff") 80313481Sgiacomo.travaglini@arm.com return data 80413481Sgiacomo.travaglini@arm.com 80513481Sgiacomo.travaglini@arm.com def _CollapseKeywords(self, content, keyword_str): 80613481Sgiacomo.travaglini@arm.com """Collapses SVN keywords.""" 80713481Sgiacomo.travaglini@arm.com # svn cat translates keywords but svn diff doesn't. As a result of this 80813481Sgiacomo.travaglini@arm.com # behavior patching.PatchChunks() fails with a chunk mismatch error. 80913481Sgiacomo.travaglini@arm.com # This part was originally written by the Review Board development team 81013481Sgiacomo.travaglini@arm.com # who had the same problem (http://reviews.review-board.org/r/276/). 81113481Sgiacomo.travaglini@arm.com # Mapping of keywords to known aliases 81213481Sgiacomo.travaglini@arm.com svn_keywords = { 81313481Sgiacomo.travaglini@arm.com # Standard keywords 81413481Sgiacomo.travaglini@arm.com 'Date': ['Date', 'LastChangedDate'], 81513481Sgiacomo.travaglini@arm.com 'Revision': ['Revision', 'LastChangedRevision', 'Rev'], 81613481Sgiacomo.travaglini@arm.com 'Author': ['Author', 'LastChangedBy'], 81713481Sgiacomo.travaglini@arm.com 'HeadURL': ['HeadURL', 'URL'], 81813481Sgiacomo.travaglini@arm.com 'Id': ['Id'], 81913481Sgiacomo.travaglini@arm.com 82013481Sgiacomo.travaglini@arm.com # Aliases 82113481Sgiacomo.travaglini@arm.com 'LastChangedDate': ['LastChangedDate', 'Date'], 82213481Sgiacomo.travaglini@arm.com 'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'], 82313481Sgiacomo.travaglini@arm.com 'LastChangedBy': ['LastChangedBy', 'Author'], 82413481Sgiacomo.travaglini@arm.com 'URL': ['URL', 'HeadURL'], 82513481Sgiacomo.travaglini@arm.com } 82613481Sgiacomo.travaglini@arm.com 82713481Sgiacomo.travaglini@arm.com def repl(m): 82813481Sgiacomo.travaglini@arm.com if m.group(2): 82913481Sgiacomo.travaglini@arm.com return "$%s::%s$" % (m.group(1), " " * len(m.group(3))) 83013481Sgiacomo.travaglini@arm.com return "$%s$" % m.group(1) 83113481Sgiacomo.travaglini@arm.com keywords = [keyword 83213481Sgiacomo.travaglini@arm.com for name in keyword_str.split(" ") 83313481Sgiacomo.travaglini@arm.com for keyword in svn_keywords.get(name, [])] 83413481Sgiacomo.travaglini@arm.com return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content) 83513481Sgiacomo.travaglini@arm.com 83613481Sgiacomo.travaglini@arm.com def GetUnknownFiles(self): 83713481Sgiacomo.travaglini@arm.com status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True) 83813481Sgiacomo.travaglini@arm.com unknown_files = [] 83913481Sgiacomo.travaglini@arm.com for line in status.split("\n"): 84013481Sgiacomo.travaglini@arm.com if line and line[0] == "?": 84113481Sgiacomo.travaglini@arm.com unknown_files.append(line) 84213481Sgiacomo.travaglini@arm.com return unknown_files 84313481Sgiacomo.travaglini@arm.com 84413481Sgiacomo.travaglini@arm.com def ReadFile(self, filename): 84513481Sgiacomo.travaglini@arm.com """Returns the contents of a file.""" 84613481Sgiacomo.travaglini@arm.com file = open(filename, 'rb') 84713481Sgiacomo.travaglini@arm.com result = "" 84813481Sgiacomo.travaglini@arm.com try: 84913481Sgiacomo.travaglini@arm.com result = file.read() 85013481Sgiacomo.travaglini@arm.com finally: 85113481Sgiacomo.travaglini@arm.com file.close() 85213481Sgiacomo.travaglini@arm.com return result 85313481Sgiacomo.travaglini@arm.com 85413481Sgiacomo.travaglini@arm.com def GetStatus(self, filename): 85513481Sgiacomo.travaglini@arm.com """Returns the status of a file.""" 85613481Sgiacomo.travaglini@arm.com if not self.options.revision: 85713481Sgiacomo.travaglini@arm.com status = RunShell(["svn", "status", "--ignore-externals", filename]) 85813481Sgiacomo.travaglini@arm.com if not status: 85913481Sgiacomo.travaglini@arm.com ErrorExit("svn status returned no output for %s" % filename) 86013481Sgiacomo.travaglini@arm.com status_lines = status.splitlines() 86113481Sgiacomo.travaglini@arm.com # If file is in a cl, the output will begin with 86213481Sgiacomo.travaglini@arm.com # "\n--- Changelist 'cl_name':\n". See 86313481Sgiacomo.travaglini@arm.com # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt 86413481Sgiacomo.travaglini@arm.com if (len(status_lines) == 3 and 86513481Sgiacomo.travaglini@arm.com not status_lines[0] and 86613481Sgiacomo.travaglini@arm.com status_lines[1].startswith("--- Changelist")): 86713481Sgiacomo.travaglini@arm.com status = status_lines[2] 86813481Sgiacomo.travaglini@arm.com else: 86913481Sgiacomo.travaglini@arm.com status = status_lines[0] 87013481Sgiacomo.travaglini@arm.com # If we have a revision to diff against we need to run "svn list" 87113481Sgiacomo.travaglini@arm.com # for the old and the new revision and compare the results to get 87213481Sgiacomo.travaglini@arm.com # the correct status for a file. 87313481Sgiacomo.travaglini@arm.com else: 87413481Sgiacomo.travaglini@arm.com dirname, relfilename = os.path.split(filename) 87513481Sgiacomo.travaglini@arm.com if dirname not in self.svnls_cache: 87613481Sgiacomo.travaglini@arm.com cmd = ["svn", "list", "-r", self.rev_start, dirname or "."] 87713481Sgiacomo.travaglini@arm.com out, returncode = RunShellWithReturnCode(cmd) 87813481Sgiacomo.travaglini@arm.com if returncode: 87913481Sgiacomo.travaglini@arm.com ErrorExit("Failed to get status for %s." % filename) 88013481Sgiacomo.travaglini@arm.com old_files = out.splitlines() 88113481Sgiacomo.travaglini@arm.com args = ["svn", "list"] 88213481Sgiacomo.travaglini@arm.com if self.rev_end: 88313481Sgiacomo.travaglini@arm.com args += ["-r", self.rev_end] 88413481Sgiacomo.travaglini@arm.com cmd = args + [dirname or "."] 88513481Sgiacomo.travaglini@arm.com out, returncode = RunShellWithReturnCode(cmd) 88613481Sgiacomo.travaglini@arm.com if returncode: 88713481Sgiacomo.travaglini@arm.com ErrorExit("Failed to run command %s" % cmd) 88813481Sgiacomo.travaglini@arm.com self.svnls_cache[dirname] = (old_files, out.splitlines()) 88913481Sgiacomo.travaglini@arm.com old_files, new_files = self.svnls_cache[dirname] 89013481Sgiacomo.travaglini@arm.com if relfilename in old_files and relfilename not in new_files: 89113481Sgiacomo.travaglini@arm.com status = "D " 89213481Sgiacomo.travaglini@arm.com elif relfilename in old_files and relfilename in new_files: 89313481Sgiacomo.travaglini@arm.com status = "M " 89413481Sgiacomo.travaglini@arm.com else: 89513481Sgiacomo.travaglini@arm.com status = "A " 89613481Sgiacomo.travaglini@arm.com return status 89713481Sgiacomo.travaglini@arm.com 89813481Sgiacomo.travaglini@arm.com def GetBaseFile(self, filename): 89913481Sgiacomo.travaglini@arm.com status = self.GetStatus(filename) 90013481Sgiacomo.travaglini@arm.com base_content = None 90113481Sgiacomo.travaglini@arm.com new_content = None 90213481Sgiacomo.travaglini@arm.com 90313481Sgiacomo.travaglini@arm.com # If a file is copied its status will be "A +", which signifies 90413481Sgiacomo.travaglini@arm.com # "addition-with-history". See "svn st" for more information. We need to 90513481Sgiacomo.travaglini@arm.com # upload the original file or else diff parsing will fail if the file was 90613481Sgiacomo.travaglini@arm.com # edited. 90713481Sgiacomo.travaglini@arm.com if status[0] == "A" and status[3] != "+": 90813481Sgiacomo.travaglini@arm.com # We'll need to upload the new content if we're adding a binary file 90913481Sgiacomo.travaglini@arm.com # since diff's output won't contain it. 91013481Sgiacomo.travaglini@arm.com mimetype = RunShell(["svn", "propget", "svn:mime-type", filename], 91113481Sgiacomo.travaglini@arm.com silent_ok=True) 91213481Sgiacomo.travaglini@arm.com base_content = "" 91313481Sgiacomo.travaglini@arm.com is_binary = mimetype and not mimetype.startswith("text/") 91413481Sgiacomo.travaglini@arm.com if is_binary and self.IsImage(filename): 91513481Sgiacomo.travaglini@arm.com new_content = self.ReadFile(filename) 91613481Sgiacomo.travaglini@arm.com elif (status[0] in ("M", "D", "R") or 91713481Sgiacomo.travaglini@arm.com (status[0] == "A" and status[3] == "+") or # Copied file. 91813481Sgiacomo.travaglini@arm.com (status[0] == " " and status[1] == "M")): # Property change. 91913481Sgiacomo.travaglini@arm.com args = [] 92013481Sgiacomo.travaglini@arm.com if self.options.revision: 92113481Sgiacomo.travaglini@arm.com url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) 92213481Sgiacomo.travaglini@arm.com else: 92313481Sgiacomo.travaglini@arm.com # Don't change filename, it's needed later. 92413481Sgiacomo.travaglini@arm.com url = filename 92513481Sgiacomo.travaglini@arm.com args += ["-r", "BASE"] 92613481Sgiacomo.travaglini@arm.com cmd = ["svn"] + args + ["propget", "svn:mime-type", url] 92713481Sgiacomo.travaglini@arm.com mimetype, returncode = RunShellWithReturnCode(cmd) 92813481Sgiacomo.travaglini@arm.com if returncode: 92913481Sgiacomo.travaglini@arm.com # File does not exist in the requested revision. 93013481Sgiacomo.travaglini@arm.com # Reset mimetype, it contains an error message. 93113481Sgiacomo.travaglini@arm.com mimetype = "" 93213481Sgiacomo.travaglini@arm.com get_base = False 93313481Sgiacomo.travaglini@arm.com is_binary = mimetype and not mimetype.startswith("text/") 93413481Sgiacomo.travaglini@arm.com if status[0] == " ": 93513481Sgiacomo.travaglini@arm.com # Empty base content just to force an upload. 93613481Sgiacomo.travaglini@arm.com base_content = "" 93713481Sgiacomo.travaglini@arm.com elif is_binary: 93813481Sgiacomo.travaglini@arm.com if self.IsImage(filename): 93913481Sgiacomo.travaglini@arm.com get_base = True 94013481Sgiacomo.travaglini@arm.com if status[0] == "M": 94113481Sgiacomo.travaglini@arm.com if not self.rev_end: 94213481Sgiacomo.travaglini@arm.com new_content = self.ReadFile(filename) 94313481Sgiacomo.travaglini@arm.com else: 94413481Sgiacomo.travaglini@arm.com url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end) 94513481Sgiacomo.travaglini@arm.com new_content = RunShell(["svn", "cat", url], 94613481Sgiacomo.travaglini@arm.com universal_newlines=True, silent_ok=True) 94713481Sgiacomo.travaglini@arm.com else: 94813481Sgiacomo.travaglini@arm.com base_content = "" 94913481Sgiacomo.travaglini@arm.com else: 95013481Sgiacomo.travaglini@arm.com get_base = True 95113481Sgiacomo.travaglini@arm.com 95213481Sgiacomo.travaglini@arm.com if get_base: 95313481Sgiacomo.travaglini@arm.com if is_binary: 95413481Sgiacomo.travaglini@arm.com universal_newlines = False 95513481Sgiacomo.travaglini@arm.com else: 95613481Sgiacomo.travaglini@arm.com universal_newlines = True 95713481Sgiacomo.travaglini@arm.com if self.rev_start: 95813481Sgiacomo.travaglini@arm.com # "svn cat -r REV delete_file.txt" doesn't work. cat requires 95913481Sgiacomo.travaglini@arm.com # the full URL with "@REV" appended instead of using "-r" option. 96013481Sgiacomo.travaglini@arm.com url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) 96113481Sgiacomo.travaglini@arm.com base_content = RunShell(["svn", "cat", url], 96213481Sgiacomo.travaglini@arm.com universal_newlines=universal_newlines, 96313481Sgiacomo.travaglini@arm.com silent_ok=True) 96413481Sgiacomo.travaglini@arm.com else: 96513481Sgiacomo.travaglini@arm.com base_content = RunShell(["svn", "cat", filename], 96613481Sgiacomo.travaglini@arm.com universal_newlines=universal_newlines, 96713481Sgiacomo.travaglini@arm.com silent_ok=True) 96813481Sgiacomo.travaglini@arm.com if not is_binary: 96913481Sgiacomo.travaglini@arm.com args = [] 97013481Sgiacomo.travaglini@arm.com if self.rev_start: 97113481Sgiacomo.travaglini@arm.com url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start) 97213481Sgiacomo.travaglini@arm.com else: 97313481Sgiacomo.travaglini@arm.com url = filename 97413481Sgiacomo.travaglini@arm.com args += ["-r", "BASE"] 97513481Sgiacomo.travaglini@arm.com cmd = ["svn"] + args + ["propget", "svn:keywords", url] 97613481Sgiacomo.travaglini@arm.com keywords, returncode = RunShellWithReturnCode(cmd) 97713481Sgiacomo.travaglini@arm.com if keywords and not returncode: 97813481Sgiacomo.travaglini@arm.com base_content = self._CollapseKeywords(base_content, keywords) 97913481Sgiacomo.travaglini@arm.com else: 98013481Sgiacomo.travaglini@arm.com StatusUpdate("svn status returned unexpected output: %s" % status) 98113481Sgiacomo.travaglini@arm.com sys.exit(1) 98213481Sgiacomo.travaglini@arm.com return base_content, new_content, is_binary, status[0:5] 98313481Sgiacomo.travaglini@arm.com 98413481Sgiacomo.travaglini@arm.com 98513481Sgiacomo.travaglini@arm.comclass GitVCS(VersionControlSystem): 98613481Sgiacomo.travaglini@arm.com """Implementation of the VersionControlSystem interface for Git.""" 98713481Sgiacomo.travaglini@arm.com 98813481Sgiacomo.travaglini@arm.com def __init__(self, options): 98913481Sgiacomo.travaglini@arm.com super(GitVCS, self).__init__(options) 99013481Sgiacomo.travaglini@arm.com # Map of filename -> hash of base file. 99113481Sgiacomo.travaglini@arm.com self.base_hashes = {} 99213481Sgiacomo.travaglini@arm.com 99313481Sgiacomo.travaglini@arm.com def GenerateDiff(self, extra_args): 99413481Sgiacomo.travaglini@arm.com # This is more complicated than svn's GenerateDiff because we must convert 99513481Sgiacomo.travaglini@arm.com # the diff output to include an svn-style "Index:" line as well as record 99613481Sgiacomo.travaglini@arm.com # the hashes of the base files, so we can upload them along with our diff. 99713481Sgiacomo.travaglini@arm.com if self.options.revision: 99813481Sgiacomo.travaglini@arm.com extra_args = [self.options.revision] + extra_args 99913481Sgiacomo.travaglini@arm.com gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args) 100013481Sgiacomo.travaglini@arm.com svndiff = [] 100113481Sgiacomo.travaglini@arm.com filecount = 0 100213481Sgiacomo.travaglini@arm.com filename = None 100313481Sgiacomo.travaglini@arm.com for line in gitdiff.splitlines(): 100413481Sgiacomo.travaglini@arm.com match = re.match(r"diff --git a/(.*) b/.*$", line) 100513481Sgiacomo.travaglini@arm.com if match: 100613481Sgiacomo.travaglini@arm.com filecount += 1 100713481Sgiacomo.travaglini@arm.com filename = match.group(1) 100813481Sgiacomo.travaglini@arm.com svndiff.append("Index: %s\n" % filename) 100913481Sgiacomo.travaglini@arm.com else: 101013481Sgiacomo.travaglini@arm.com # The "index" line in a git diff looks like this (long hashes elided): 101113481Sgiacomo.travaglini@arm.com # index 82c0d44..b2cee3f 100755 101213481Sgiacomo.travaglini@arm.com # We want to save the left hash, as that identifies the base file. 101313481Sgiacomo.travaglini@arm.com match = re.match(r"index (\w+)\.\.", line) 101413481Sgiacomo.travaglini@arm.com if match: 101513481Sgiacomo.travaglini@arm.com self.base_hashes[filename] = match.group(1) 101613481Sgiacomo.travaglini@arm.com svndiff.append(line + "\n") 101713481Sgiacomo.travaglini@arm.com if not filecount: 101813481Sgiacomo.travaglini@arm.com ErrorExit("No valid patches found in output from git diff") 101913481Sgiacomo.travaglini@arm.com return "".join(svndiff) 102013481Sgiacomo.travaglini@arm.com 102113481Sgiacomo.travaglini@arm.com def GetUnknownFiles(self): 102213481Sgiacomo.travaglini@arm.com status = RunShell(["git", "ls-files", "--exclude-standard", "--others"], 102313481Sgiacomo.travaglini@arm.com silent_ok=True) 102413481Sgiacomo.travaglini@arm.com return status.splitlines() 102513481Sgiacomo.travaglini@arm.com 102613481Sgiacomo.travaglini@arm.com def GetBaseFile(self, filename): 102713481Sgiacomo.travaglini@arm.com hash = self.base_hashes[filename] 102813481Sgiacomo.travaglini@arm.com base_content = None 102913481Sgiacomo.travaglini@arm.com new_content = None 103013481Sgiacomo.travaglini@arm.com is_binary = False 103113481Sgiacomo.travaglini@arm.com if hash == "0" * 40: # All-zero hash indicates no base file. 103213481Sgiacomo.travaglini@arm.com status = "A" 103313481Sgiacomo.travaglini@arm.com base_content = "" 103413481Sgiacomo.travaglini@arm.com else: 103513481Sgiacomo.travaglini@arm.com status = "M" 103613481Sgiacomo.travaglini@arm.com base_content, returncode = RunShellWithReturnCode(["git", "show", hash]) 103713481Sgiacomo.travaglini@arm.com if returncode: 103813481Sgiacomo.travaglini@arm.com ErrorExit("Got error status from 'git show %s'" % hash) 103913481Sgiacomo.travaglini@arm.com return (base_content, new_content, is_binary, status) 104013481Sgiacomo.travaglini@arm.com 104113481Sgiacomo.travaglini@arm.com 104213481Sgiacomo.travaglini@arm.comclass MercurialVCS(VersionControlSystem): 104313481Sgiacomo.travaglini@arm.com """Implementation of the VersionControlSystem interface for Mercurial.""" 104413481Sgiacomo.travaglini@arm.com 104513481Sgiacomo.travaglini@arm.com def __init__(self, options, repo_dir): 104613481Sgiacomo.travaglini@arm.com super(MercurialVCS, self).__init__(options) 104713481Sgiacomo.travaglini@arm.com # Absolute path to repository (we can be in a subdir) 104813481Sgiacomo.travaglini@arm.com self.repo_dir = os.path.normpath(repo_dir) 104913481Sgiacomo.travaglini@arm.com # Compute the subdir 105013481Sgiacomo.travaglini@arm.com cwd = os.path.normpath(os.getcwd()) 105113481Sgiacomo.travaglini@arm.com assert cwd.startswith(self.repo_dir) 105213481Sgiacomo.travaglini@arm.com self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/") 105313481Sgiacomo.travaglini@arm.com if self.options.revision: 105413481Sgiacomo.travaglini@arm.com self.base_rev = self.options.revision 105513481Sgiacomo.travaglini@arm.com else: 105613481Sgiacomo.travaglini@arm.com self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip() 105713481Sgiacomo.travaglini@arm.com 105813481Sgiacomo.travaglini@arm.com def _GetRelPath(self, filename): 105913481Sgiacomo.travaglini@arm.com """Get relative path of a file according to the current directory, 106013481Sgiacomo.travaglini@arm.com given its logical path in the repo.""" 106113481Sgiacomo.travaglini@arm.com assert filename.startswith(self.subdir), filename 106213481Sgiacomo.travaglini@arm.com return filename[len(self.subdir):].lstrip(r"\/") 106313481Sgiacomo.travaglini@arm.com 106413481Sgiacomo.travaglini@arm.com def GenerateDiff(self, extra_args): 106513481Sgiacomo.travaglini@arm.com # If no file specified, restrict to the current subdir 106613481Sgiacomo.travaglini@arm.com extra_args = extra_args or ["."] 106713481Sgiacomo.travaglini@arm.com cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args 106813481Sgiacomo.travaglini@arm.com data = RunShell(cmd, silent_ok=True) 106913481Sgiacomo.travaglini@arm.com svndiff = [] 107013481Sgiacomo.travaglini@arm.com filecount = 0 107113481Sgiacomo.travaglini@arm.com for line in data.splitlines(): 107213481Sgiacomo.travaglini@arm.com m = re.match("diff --git a/(\S+) b/(\S+)", line) 107313481Sgiacomo.travaglini@arm.com if m: 107413481Sgiacomo.travaglini@arm.com # Modify line to make it look like as it comes from svn diff. 107513481Sgiacomo.travaglini@arm.com # With this modification no changes on the server side are required 107613481Sgiacomo.travaglini@arm.com # to make upload.py work with Mercurial repos. 107713481Sgiacomo.travaglini@arm.com # NOTE: for proper handling of moved/copied files, we have to use 107813481Sgiacomo.travaglini@arm.com # the second filename. 107913481Sgiacomo.travaglini@arm.com filename = m.group(2) 108013481Sgiacomo.travaglini@arm.com svndiff.append("Index: %s" % filename) 108113481Sgiacomo.travaglini@arm.com svndiff.append("=" * 67) 108213481Sgiacomo.travaglini@arm.com filecount += 1 108313481Sgiacomo.travaglini@arm.com logging.info(line) 108413481Sgiacomo.travaglini@arm.com else: 108513481Sgiacomo.travaglini@arm.com svndiff.append(line) 108613481Sgiacomo.travaglini@arm.com if not filecount: 108713481Sgiacomo.travaglini@arm.com ErrorExit("No valid patches found in output from hg diff") 108813481Sgiacomo.travaglini@arm.com return "\n".join(svndiff) + "\n" 108913481Sgiacomo.travaglini@arm.com 109013481Sgiacomo.travaglini@arm.com def GetUnknownFiles(self): 109113481Sgiacomo.travaglini@arm.com """Return a list of files unknown to the VCS.""" 109213481Sgiacomo.travaglini@arm.com args = [] 109313481Sgiacomo.travaglini@arm.com status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."], 109413481Sgiacomo.travaglini@arm.com silent_ok=True) 109513481Sgiacomo.travaglini@arm.com unknown_files = [] 109613481Sgiacomo.travaglini@arm.com for line in status.splitlines(): 109713481Sgiacomo.travaglini@arm.com st, fn = line.split(" ", 1) 109813481Sgiacomo.travaglini@arm.com if st == "?": 109913481Sgiacomo.travaglini@arm.com unknown_files.append(fn) 110013481Sgiacomo.travaglini@arm.com return unknown_files 110113481Sgiacomo.travaglini@arm.com 110213481Sgiacomo.travaglini@arm.com def GetBaseFile(self, filename): 110313481Sgiacomo.travaglini@arm.com # "hg status" and "hg cat" both take a path relative to the current subdir 110413481Sgiacomo.travaglini@arm.com # rather than to the repo root, but "hg diff" has given us the full path 110513481Sgiacomo.travaglini@arm.com # to the repo root. 110613481Sgiacomo.travaglini@arm.com base_content = "" 110713481Sgiacomo.travaglini@arm.com new_content = None 110813481Sgiacomo.travaglini@arm.com is_binary = False 110913481Sgiacomo.travaglini@arm.com oldrelpath = relpath = self._GetRelPath(filename) 111013481Sgiacomo.travaglini@arm.com # "hg status -C" returns two lines for moved/copied files, one otherwise 111113481Sgiacomo.travaglini@arm.com out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath]) 111213481Sgiacomo.travaglini@arm.com out = out.splitlines() 111313481Sgiacomo.travaglini@arm.com # HACK: strip error message about missing file/directory if it isn't in 111413481Sgiacomo.travaglini@arm.com # the working copy 111513481Sgiacomo.travaglini@arm.com if out[0].startswith('%s: ' % relpath): 111613481Sgiacomo.travaglini@arm.com out = out[1:] 111713481Sgiacomo.travaglini@arm.com if len(out) > 1: 111813481Sgiacomo.travaglini@arm.com # Moved/copied => considered as modified, use old filename to 111913481Sgiacomo.travaglini@arm.com # retrieve base contents 112013481Sgiacomo.travaglini@arm.com oldrelpath = out[1].strip() 112113481Sgiacomo.travaglini@arm.com status = "M" 112213481Sgiacomo.travaglini@arm.com else: 112313481Sgiacomo.travaglini@arm.com status, _ = out[0].split(' ', 1) 112413481Sgiacomo.travaglini@arm.com if status != "A": 112513481Sgiacomo.travaglini@arm.com base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath], 112613481Sgiacomo.travaglini@arm.com silent_ok=True) 112713481Sgiacomo.travaglini@arm.com is_binary = "\0" in base_content # Mercurial's heuristic 112813481Sgiacomo.travaglini@arm.com if status != "R": 112913481Sgiacomo.travaglini@arm.com new_content = open(relpath, "rb").read() 113013481Sgiacomo.travaglini@arm.com is_binary = is_binary or "\0" in new_content 113113481Sgiacomo.travaglini@arm.com if is_binary and base_content: 113213481Sgiacomo.travaglini@arm.com # Fetch again without converting newlines 113313481Sgiacomo.travaglini@arm.com base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath], 113413481Sgiacomo.travaglini@arm.com silent_ok=True, universal_newlines=False) 113513481Sgiacomo.travaglini@arm.com if not is_binary or not self.IsImage(relpath): 113613481Sgiacomo.travaglini@arm.com new_content = None 113713481Sgiacomo.travaglini@arm.com return base_content, new_content, is_binary, status 113813481Sgiacomo.travaglini@arm.com 113913481Sgiacomo.travaglini@arm.com 114013481Sgiacomo.travaglini@arm.com# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync. 114113481Sgiacomo.travaglini@arm.comdef SplitPatch(data): 114213481Sgiacomo.travaglini@arm.com """Splits a patch into separate pieces for each file. 114313481Sgiacomo.travaglini@arm.com 114413481Sgiacomo.travaglini@arm.com Args: 114513481Sgiacomo.travaglini@arm.com data: A string containing the output of svn diff. 114613481Sgiacomo.travaglini@arm.com 114713481Sgiacomo.travaglini@arm.com Returns: 114813481Sgiacomo.travaglini@arm.com A list of 2-tuple (filename, text) where text is the svn diff output 114913481Sgiacomo.travaglini@arm.com pertaining to filename. 115013481Sgiacomo.travaglini@arm.com """ 115113481Sgiacomo.travaglini@arm.com patches = [] 115213481Sgiacomo.travaglini@arm.com filename = None 115313481Sgiacomo.travaglini@arm.com diff = [] 115413481Sgiacomo.travaglini@arm.com for line in data.splitlines(True): 115513481Sgiacomo.travaglini@arm.com new_filename = None 115613481Sgiacomo.travaglini@arm.com if line.startswith('Index:'): 115713481Sgiacomo.travaglini@arm.com unused, new_filename = line.split(':', 1) 115813481Sgiacomo.travaglini@arm.com new_filename = new_filename.strip() 115913481Sgiacomo.travaglini@arm.com elif line.startswith('Property changes on:'): 116013481Sgiacomo.travaglini@arm.com unused, temp_filename = line.split(':', 1) 116113481Sgiacomo.travaglini@arm.com # When a file is modified, paths use '/' between directories, however 116213481Sgiacomo.travaglini@arm.com # when a property is modified '\' is used on Windows. Make them the same 116313481Sgiacomo.travaglini@arm.com # otherwise the file shows up twice. 116413481Sgiacomo.travaglini@arm.com temp_filename = temp_filename.strip().replace('\\', '/') 116513481Sgiacomo.travaglini@arm.com if temp_filename != filename: 116613481Sgiacomo.travaglini@arm.com # File has property changes but no modifications, create a new diff. 116713481Sgiacomo.travaglini@arm.com new_filename = temp_filename 116813481Sgiacomo.travaglini@arm.com if new_filename: 116913481Sgiacomo.travaglini@arm.com if filename and diff: 117013481Sgiacomo.travaglini@arm.com patches.append((filename, ''.join(diff))) 117113481Sgiacomo.travaglini@arm.com filename = new_filename 117213481Sgiacomo.travaglini@arm.com diff = [line] 117313481Sgiacomo.travaglini@arm.com continue 117413481Sgiacomo.travaglini@arm.com if diff is not None: 117513481Sgiacomo.travaglini@arm.com diff.append(line) 117613481Sgiacomo.travaglini@arm.com if filename and diff: 117713481Sgiacomo.travaglini@arm.com patches.append((filename, ''.join(diff))) 117813481Sgiacomo.travaglini@arm.com return patches 117913481Sgiacomo.travaglini@arm.com 118013481Sgiacomo.travaglini@arm.com 118113481Sgiacomo.travaglini@arm.comdef UploadSeparatePatches(issue, rpc_server, patchset, data, options): 118213481Sgiacomo.travaglini@arm.com """Uploads a separate patch for each file in the diff output. 118313481Sgiacomo.travaglini@arm.com 118413481Sgiacomo.travaglini@arm.com Returns a list of [patch_key, filename] for each file. 118513481Sgiacomo.travaglini@arm.com """ 118613481Sgiacomo.travaglini@arm.com patches = SplitPatch(data) 118713481Sgiacomo.travaglini@arm.com rv = [] 118813481Sgiacomo.travaglini@arm.com for patch in patches: 118913481Sgiacomo.travaglini@arm.com if len(patch[1]) > MAX_UPLOAD_SIZE: 119013481Sgiacomo.travaglini@arm.com print ("Not uploading the patch for " + patch[0] + 119113481Sgiacomo.travaglini@arm.com " because the file is too large.") 119213481Sgiacomo.travaglini@arm.com continue 119313481Sgiacomo.travaglini@arm.com form_fields = [("filename", patch[0])] 119413481Sgiacomo.travaglini@arm.com if not options.download_base: 119513481Sgiacomo.travaglini@arm.com form_fields.append(("content_upload", "1")) 119613481Sgiacomo.travaglini@arm.com files = [("data", "data.diff", patch[1])] 119713481Sgiacomo.travaglini@arm.com ctype, body = EncodeMultipartFormData(form_fields, files) 119813481Sgiacomo.travaglini@arm.com url = "/%d/upload_patch/%d" % (int(issue), int(patchset)) 119913481Sgiacomo.travaglini@arm.com print "Uploading patch for " + patch[0] 120013481Sgiacomo.travaglini@arm.com response_body = rpc_server.Send(url, body, content_type=ctype) 120113481Sgiacomo.travaglini@arm.com lines = response_body.splitlines() 120213481Sgiacomo.travaglini@arm.com if not lines or lines[0] != "OK": 120313481Sgiacomo.travaglini@arm.com StatusUpdate(" --> %s" % response_body) 120413481Sgiacomo.travaglini@arm.com sys.exit(1) 120513481Sgiacomo.travaglini@arm.com rv.append([lines[1], patch[0]]) 120613481Sgiacomo.travaglini@arm.com return rv 120713481Sgiacomo.travaglini@arm.com 120813481Sgiacomo.travaglini@arm.com 120913481Sgiacomo.travaglini@arm.comdef GuessVCS(options): 121013481Sgiacomo.travaglini@arm.com """Helper to guess the version control system. 121113481Sgiacomo.travaglini@arm.com 121213481Sgiacomo.travaglini@arm.com This examines the current directory, guesses which VersionControlSystem 121313481Sgiacomo.travaglini@arm.com we're using, and returns an instance of the appropriate class. Exit with an 121413481Sgiacomo.travaglini@arm.com error if we can't figure it out. 121513481Sgiacomo.travaglini@arm.com 121613481Sgiacomo.travaglini@arm.com Returns: 121713481Sgiacomo.travaglini@arm.com A VersionControlSystem instance. Exits if the VCS can't be guessed. 121813481Sgiacomo.travaglini@arm.com """ 121913481Sgiacomo.travaglini@arm.com # Mercurial has a command to get the base directory of a repository 122013481Sgiacomo.travaglini@arm.com # Try running it, but don't die if we don't have hg installed. 122113481Sgiacomo.travaglini@arm.com # NOTE: we try Mercurial first as it can sit on top of an SVN working copy. 122213481Sgiacomo.travaglini@arm.com try: 122313481Sgiacomo.travaglini@arm.com out, returncode = RunShellWithReturnCode(["hg", "root"]) 122413481Sgiacomo.travaglini@arm.com if returncode == 0: 122513481Sgiacomo.travaglini@arm.com return MercurialVCS(options, out.strip()) 122613481Sgiacomo.travaglini@arm.com except OSError, (errno, message): 122713481Sgiacomo.travaglini@arm.com if errno != 2: # ENOENT -- they don't have hg installed. 122813481Sgiacomo.travaglini@arm.com raise 122913481Sgiacomo.travaglini@arm.com 123013481Sgiacomo.travaglini@arm.com # Subversion has a .svn in all working directories. 123113481Sgiacomo.travaglini@arm.com if os.path.isdir('.svn'): 123213481Sgiacomo.travaglini@arm.com logging.info("Guessed VCS = Subversion") 123313481Sgiacomo.travaglini@arm.com return SubversionVCS(options) 123413481Sgiacomo.travaglini@arm.com 123513481Sgiacomo.travaglini@arm.com # Git has a command to test if you're in a git tree. 123613481Sgiacomo.travaglini@arm.com # Try running it, but don't die if we don't have git installed. 123713481Sgiacomo.travaglini@arm.com try: 123813481Sgiacomo.travaglini@arm.com out, returncode = RunShellWithReturnCode(["git", "rev-parse", 123913481Sgiacomo.travaglini@arm.com "--is-inside-work-tree"]) 124013481Sgiacomo.travaglini@arm.com if returncode == 0: 124113481Sgiacomo.travaglini@arm.com return GitVCS(options) 124213481Sgiacomo.travaglini@arm.com except OSError, (errno, message): 124313481Sgiacomo.travaglini@arm.com if errno != 2: # ENOENT -- they don't have git installed. 124413481Sgiacomo.travaglini@arm.com raise 124513481Sgiacomo.travaglini@arm.com 124613481Sgiacomo.travaglini@arm.com ErrorExit(("Could not guess version control system. " 124713481Sgiacomo.travaglini@arm.com "Are you in a working copy directory?")) 124813481Sgiacomo.travaglini@arm.com 124913481Sgiacomo.travaglini@arm.com 125013481Sgiacomo.travaglini@arm.comdef RealMain(argv, data=None): 125113481Sgiacomo.travaglini@arm.com """The real main function. 125213481Sgiacomo.travaglini@arm.com 125313481Sgiacomo.travaglini@arm.com Args: 125413481Sgiacomo.travaglini@arm.com argv: Command line arguments. 125513481Sgiacomo.travaglini@arm.com data: Diff contents. If None (default) the diff is generated by 125613481Sgiacomo.travaglini@arm.com the VersionControlSystem implementation returned by GuessVCS(). 125713481Sgiacomo.travaglini@arm.com 125813481Sgiacomo.travaglini@arm.com Returns: 125913481Sgiacomo.travaglini@arm.com A 2-tuple (issue id, patchset id). 126013481Sgiacomo.travaglini@arm.com The patchset id is None if the base files are not uploaded by this 126113481Sgiacomo.travaglini@arm.com script (applies only to SVN checkouts). 126213481Sgiacomo.travaglini@arm.com """ 126313481Sgiacomo.travaglini@arm.com logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:" 126413481Sgiacomo.travaglini@arm.com "%(lineno)s %(message)s ")) 126513481Sgiacomo.travaglini@arm.com os.environ['LC_ALL'] = 'C' 126613481Sgiacomo.travaglini@arm.com options, args = parser.parse_args(argv[1:]) 126713481Sgiacomo.travaglini@arm.com global verbosity 126813481Sgiacomo.travaglini@arm.com verbosity = options.verbose 126913481Sgiacomo.travaglini@arm.com if verbosity >= 3: 127013481Sgiacomo.travaglini@arm.com logging.getLogger().setLevel(logging.DEBUG) 127113481Sgiacomo.travaglini@arm.com elif verbosity >= 2: 127213481Sgiacomo.travaglini@arm.com logging.getLogger().setLevel(logging.INFO) 127313481Sgiacomo.travaglini@arm.com vcs = GuessVCS(options) 127413481Sgiacomo.travaglini@arm.com if isinstance(vcs, SubversionVCS): 127513481Sgiacomo.travaglini@arm.com # base field is only allowed for Subversion. 127613481Sgiacomo.travaglini@arm.com # Note: Fetching base files may become deprecated in future releases. 127713481Sgiacomo.travaglini@arm.com base = vcs.GuessBase(options.download_base) 127813481Sgiacomo.travaglini@arm.com else: 127913481Sgiacomo.travaglini@arm.com base = None 128013481Sgiacomo.travaglini@arm.com if not base and options.download_base: 128113481Sgiacomo.travaglini@arm.com options.download_base = True 128213481Sgiacomo.travaglini@arm.com logging.info("Enabled upload of base file") 128313481Sgiacomo.travaglini@arm.com if not options.assume_yes: 128413481Sgiacomo.travaglini@arm.com vcs.CheckForUnknownFiles() 128513481Sgiacomo.travaglini@arm.com if data is None: 128613481Sgiacomo.travaglini@arm.com data = vcs.GenerateDiff(args) 128713481Sgiacomo.travaglini@arm.com files = vcs.GetBaseFiles(data) 128813481Sgiacomo.travaglini@arm.com if verbosity >= 1: 128913481Sgiacomo.travaglini@arm.com print "Upload server:", options.server, "(change with -s/--server)" 129013481Sgiacomo.travaglini@arm.com if options.issue: 129113481Sgiacomo.travaglini@arm.com prompt = "Message describing this patch set: " 129213481Sgiacomo.travaglini@arm.com else: 129313481Sgiacomo.travaglini@arm.com prompt = "New issue subject: " 129413481Sgiacomo.travaglini@arm.com message = options.message or raw_input(prompt).strip() 129513481Sgiacomo.travaglini@arm.com if not message: 129613481Sgiacomo.travaglini@arm.com ErrorExit("A non-empty message is required") 129713481Sgiacomo.travaglini@arm.com rpc_server = GetRpcServer(options) 129813481Sgiacomo.travaglini@arm.com form_fields = [("subject", message)] 129913481Sgiacomo.travaglini@arm.com if base: 130013481Sgiacomo.travaglini@arm.com form_fields.append(("base", base)) 130113481Sgiacomo.travaglini@arm.com if options.issue: 130213481Sgiacomo.travaglini@arm.com form_fields.append(("issue", str(options.issue))) 130313481Sgiacomo.travaglini@arm.com if options.email: 130413481Sgiacomo.travaglini@arm.com form_fields.append(("user", options.email)) 130513481Sgiacomo.travaglini@arm.com if options.reviewers: 130613481Sgiacomo.travaglini@arm.com for reviewer in options.reviewers.split(','): 130713481Sgiacomo.travaglini@arm.com if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1: 130813481Sgiacomo.travaglini@arm.com ErrorExit("Invalid email address: %s" % reviewer) 130913481Sgiacomo.travaglini@arm.com form_fields.append(("reviewers", options.reviewers)) 131013481Sgiacomo.travaglini@arm.com if options.cc: 131113481Sgiacomo.travaglini@arm.com for cc in options.cc.split(','): 131213481Sgiacomo.travaglini@arm.com if "@" in cc and not cc.split("@")[1].count(".") == 1: 131313481Sgiacomo.travaglini@arm.com ErrorExit("Invalid email address: %s" % cc) 131413481Sgiacomo.travaglini@arm.com form_fields.append(("cc", options.cc)) 131513481Sgiacomo.travaglini@arm.com description = options.description 131613481Sgiacomo.travaglini@arm.com if options.description_file: 131713481Sgiacomo.travaglini@arm.com if options.description: 131813481Sgiacomo.travaglini@arm.com ErrorExit("Can't specify description and description_file") 131913481Sgiacomo.travaglini@arm.com file = open(options.description_file, 'r') 132013481Sgiacomo.travaglini@arm.com description = file.read() 132113481Sgiacomo.travaglini@arm.com file.close() 132213481Sgiacomo.travaglini@arm.com if description: 132313481Sgiacomo.travaglini@arm.com form_fields.append(("description", description)) 132413481Sgiacomo.travaglini@arm.com # Send a hash of all the base file so the server can determine if a copy 132513481Sgiacomo.travaglini@arm.com # already exists in an earlier patchset. 132613481Sgiacomo.travaglini@arm.com base_hashes = "" 132713481Sgiacomo.travaglini@arm.com for file, info in files.iteritems(): 132813481Sgiacomo.travaglini@arm.com if not info[0] is None: 132913481Sgiacomo.travaglini@arm.com checksum = md5.new(info[0]).hexdigest() 133013481Sgiacomo.travaglini@arm.com if base_hashes: 133113481Sgiacomo.travaglini@arm.com base_hashes += "|" 133213481Sgiacomo.travaglini@arm.com base_hashes += checksum + ":" + file 133313481Sgiacomo.travaglini@arm.com form_fields.append(("base_hashes", base_hashes)) 133413481Sgiacomo.travaglini@arm.com # If we're uploading base files, don't send the email before the uploads, so 133513481Sgiacomo.travaglini@arm.com # that it contains the file status. 133613481Sgiacomo.travaglini@arm.com if options.send_mail and options.download_base: 133713481Sgiacomo.travaglini@arm.com form_fields.append(("send_mail", "1")) 133813481Sgiacomo.travaglini@arm.com if not options.download_base: 133913481Sgiacomo.travaglini@arm.com form_fields.append(("content_upload", "1")) 134013481Sgiacomo.travaglini@arm.com if len(data) > MAX_UPLOAD_SIZE: 134113481Sgiacomo.travaglini@arm.com print "Patch is large, so uploading file patches separately." 134213481Sgiacomo.travaglini@arm.com uploaded_diff_file = [] 134313481Sgiacomo.travaglini@arm.com form_fields.append(("separate_patches", "1")) 134413481Sgiacomo.travaglini@arm.com else: 134513481Sgiacomo.travaglini@arm.com uploaded_diff_file = [("data", "data.diff", data)] 134613481Sgiacomo.travaglini@arm.com ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file) 134713481Sgiacomo.travaglini@arm.com response_body = rpc_server.Send("/upload", body, content_type=ctype) 134813481Sgiacomo.travaglini@arm.com patchset = None 134913481Sgiacomo.travaglini@arm.com if not options.download_base or not uploaded_diff_file: 135013481Sgiacomo.travaglini@arm.com lines = response_body.splitlines() 135113481Sgiacomo.travaglini@arm.com if len(lines) >= 2: 135213481Sgiacomo.travaglini@arm.com msg = lines[0] 135313481Sgiacomo.travaglini@arm.com patchset = lines[1].strip() 135413481Sgiacomo.travaglini@arm.com patches = [x.split(" ", 1) for x in lines[2:]] 135513481Sgiacomo.travaglini@arm.com else: 135613481Sgiacomo.travaglini@arm.com msg = response_body 135713481Sgiacomo.travaglini@arm.com else: 135813481Sgiacomo.travaglini@arm.com msg = response_body 135913481Sgiacomo.travaglini@arm.com StatusUpdate(msg) 136013481Sgiacomo.travaglini@arm.com if not response_body.startswith("Issue created.") and \ 136113481Sgiacomo.travaglini@arm.com not response_body.startswith("Issue updated."): 136213481Sgiacomo.travaglini@arm.com sys.exit(0) 136313481Sgiacomo.travaglini@arm.com issue = msg[msg.rfind("/")+1:] 136413481Sgiacomo.travaglini@arm.com 136513481Sgiacomo.travaglini@arm.com if not uploaded_diff_file: 136613481Sgiacomo.travaglini@arm.com result = UploadSeparatePatches(issue, rpc_server, patchset, data, options) 136713481Sgiacomo.travaglini@arm.com if not options.download_base: 136813481Sgiacomo.travaglini@arm.com patches = result 136913481Sgiacomo.travaglini@arm.com 137013481Sgiacomo.travaglini@arm.com if not options.download_base: 137113481Sgiacomo.travaglini@arm.com vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files) 137213481Sgiacomo.travaglini@arm.com if options.send_mail: 137313481Sgiacomo.travaglini@arm.com rpc_server.Send("/" + issue + "/mail", payload="") 137413481Sgiacomo.travaglini@arm.com return issue, patchset 137513481Sgiacomo.travaglini@arm.com 137613481Sgiacomo.travaglini@arm.com 137713481Sgiacomo.travaglini@arm.comdef main(): 137813481Sgiacomo.travaglini@arm.com try: 137913481Sgiacomo.travaglini@arm.com RealMain(sys.argv) 138013481Sgiacomo.travaglini@arm.com except KeyboardInterrupt: 138113481Sgiacomo.travaglini@arm.com print 138213481Sgiacomo.travaglini@arm.com StatusUpdate("Interrupted.") 138313481Sgiacomo.travaglini@arm.com sys.exit(1) 138413481Sgiacomo.travaglini@arm.com 138513481Sgiacomo.travaglini@arm.com 138613481Sgiacomo.travaglini@arm.comif __name__ == "__main__": 138713481Sgiacomo.travaglini@arm.com main() 1388