1 #!/usr/bin/env python
2 #
3 # Copyright 2007 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 
17 """Tool for uploading diffs from a version control system to the codereview app.
18 
19 Usage summary: upload.py [options] [-- diff_options]
20 
21 Diff options are passed to the diff command of the underlying system.
22 
23 Supported version control systems:
24   Git
25   Mercurial
26   Subversion
27 
28 It is important for Git/Mercurial users to specify a tree/node/branch to diff
29 against by using the '--rev' option.
30 """
31 # This code is derived from appcfg.py in the App Engine SDK (open source),
32 # and from ASPN recipe #146306.
33 
34 import cookielib
35 import getpass
36 import logging
37 import mimetypes
38 import optparse
39 import os
40 import re
41 import socket
42 import subprocess
43 import sys
44 import urllib
45 import urllib2
46 import urlparse
47 
48 # The md5 module was deprecated in Python 2.5.
49 try:
50   from hashlib import md5
51 except ImportError:
52   from md5 import md5
53 
54 try:
55   import readline
56 except ImportError:
57   pass
58 
59 # The logging verbosity:
60 #  0: Errors only.
61 #  1: Status messages.
62 #  2: Info logs.
63 #  3: Debug logs.
64 verbosity = 1
65 
66 # Max size of patch or base file.
67 MAX_UPLOAD_SIZE = 900 * 1024
68 
69 # Constants for version control names.  Used by GuessVCSName.
70 VCS_GIT = "Git"
71 VCS_MERCURIAL = "Mercurial"
72 VCS_SUBVERSION = "Subversion"
73 VCS_UNKNOWN = "Unknown"
74 
75 
76 def GetEmail(prompt):
77   """Prompts the user for their email address and returns it.
78 
79   The last used email address is saved to a file and offered up as a suggestion
80   to the user. If the user presses enter without typing in anything the last
81   used email address is used. If the user enters a new address, it is saved
82   for next time we prompt.
83 
84   """
85   last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
86   last_email = ""
87   if os.path.exists(last_email_file_name):
88     try:
89       last_email_file = open(last_email_file_name, "r")
90       last_email = last_email_file.readline().strip("\n")
91       last_email_file.close()
92       prompt += " [%s]" % last_email
93     except IOError, e:
94       pass
95   email = raw_input(prompt + ": ").strip()
96   if email:
97     try:
98       last_email_file = open(last_email_file_name, "w")
99       last_email_file.write(email)
100       last_email_file.close()
101     except IOError, e:
102       pass
103   else:
104     email = last_email
105   return email
106 
107 
108 def StatusUpdate(msg):
109   """Print a status message to stdout.
110 
111   If 'verbosity' is greater than 0, print the message.
112 
113   Args:
114     msg: The string to print.
115   """
116   if verbosity > 0:
117     print msg
118 
119 
120 def ErrorExit(msg):
121   """Print an error message to stderr and exit."""
122   print >>sys.stderr, msg
123   sys.exit(1)
124 
125 
126 class ClientLoginError(urllib2.HTTPError):
127   """Raised to indicate there was an error authenticating with ClientLogin."""
128 
129   def __init__(self, url, code, msg, headers, args):
130     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
131     self.args = args
132     self.reason = args["Error"]
133 
134 
135 class AbstractRpcServer(object):
136   """Provides a common interface for a simple RPC server."""
137 
138   def __init__(self, host, auth_function, host_override=None, extra_headers={},
139                save_cookies=False):
140     """Creates a new HttpRpcServer.
141 
142     Args:
143       host: The host to send requests to.
144       auth_function: A function that takes no arguments and returns an
145         (email, password) tuple when called. Will be called if authentication
146         is required.
147       host_override: The host header to send to the server (defaults to host).
148       extra_headers: A dict of extra headers to append to every request.
149       save_cookies: If True, save the authentication cookies to local disk.
150         If False, use an in-memory cookiejar instead.  Subclasses must
151         implement this functionality.  Defaults to False.
152     """
153     self.host = host
154     self.host_override = host_override
155     self.auth_function = auth_function
156     self.authenticated = False
157     self.extra_headers = extra_headers
158     self.save_cookies = save_cookies
159     self.opener = self._GetOpener()
160     if self.host_override:
161       logging.info("Server: %s; Host: %s", self.host, self.host_override)
162     else:
163       logging.info("Server: %s", self.host)
164 
165   def _GetOpener(self):
166     """Returns an OpenerDirector for making HTTP requests.
167 
168     Returns:
169       A urllib2.OpenerDirector object.
170     """
171     raise NotImplementedError()
172 
173   def _CreateRequest(self, url, data=None):
174     """Creates a new urllib request."""
175     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
176     req = urllib2.Request(url, data=data)
177     if self.host_override:
178       req.add_header("Host", self.host_override)
179     for key, value in self.extra_headers.iteritems():
180       req.add_header(key, value)
181     return req
182 
183   def _GetAuthToken(self, email, password):
184     """Uses ClientLogin to authenticate the user, returning an auth token.
185 
186     Args:
187       email:    The user's email address
188       password: The user's password
189 
190     Raises:
191       ClientLoginError: If there was an error authenticating with ClientLogin.
192       HTTPError: If there was some other form of HTTP error.
193 
194     Returns:
195       The authentication token returned by ClientLogin.
196     """
197     account_type = "GOOGLE"
198     if self.host.endswith(".google.com"):
199       # Needed for use inside Google.
200       account_type = "HOSTED"
201     req = self._CreateRequest(
202         url="https://www.google.com/accounts/ClientLogin",
203         data=urllib.urlencode({
204             "Email": email,
205             "Passwd": password,
206             "service": "ah",
207             "source": "rietveld-codereview-upload",
208             "accountType": account_type,
209         }),
210     )
211     try:
212       response = self.opener.open(req)
213       response_body = response.read()
214       response_dict = dict(x.split("=")
215                            for x in response_body.split("\n") if x)
216       return response_dict["Auth"]
217     except urllib2.HTTPError, e:
218       if e.code == 403:
219         body = e.read()
220         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
221         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
222                                e.headers, response_dict)
223       else:
224         raise
225 
226   def _GetAuthCookie(self, auth_token):
227     """Fetches authentication cookies for an authentication token.
228 
229     Args:
230       auth_token: The authentication token returned by ClientLogin.
231 
232     Raises:
233       HTTPError: If there was an error fetching the authentication cookies.
234     """
235     # This is a dummy value to allow us to identify when we're successful.
236     continue_location = "http://localhost/"
237     args = {"continue": continue_location, "auth": auth_token}
238     req = self._CreateRequest("http://%s/_ah/login?%s" %
239                               (self.host, urllib.urlencode(args)))
240     try:
241       response = self.opener.open(req)
242     except urllib2.HTTPError, e:
243       response = e
244     if (response.code != 302 or
245         response.info()["location"] != continue_location):
246       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
247                               response.headers, response.fp)
248     self.authenticated = True
249 
250   def _Authenticate(self):
251     """Authenticates the user.
252 
253     The authentication process works as follows:
254      1) We get a username and password from the user
255      2) We use ClientLogin to obtain an AUTH token for the user
256         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
257      3) We pass the auth token to /_ah/login on the server to obtain an
258         authentication cookie. If login was successful, it tries to redirect
259         us to the URL we provided.
260 
261     If we attempt to access the upload API without first obtaining an
262     authentication cookie, it returns a 401 response and directs us to
263     authenticate ourselves with ClientLogin.
264     """
265     for i in range(3):
266       credentials = self.auth_function()
267       try:
268         auth_token = self._GetAuthToken(credentials[0], credentials[1])
269       except ClientLoginError, e:
270         if e.reason == "BadAuthentication":
271           print >>sys.stderr, "Invalid username or password."
272           continue
273         if e.reason == "CaptchaRequired":
274           print >>sys.stderr, (
275               "Please go to\n"
276               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
277               "and verify you are a human.  Then try again.")
278           break
279         if e.reason == "NotVerified":
280           print >>sys.stderr, "Account not verified."
281           break
282         if e.reason == "TermsNotAgreed":
283           print >>sys.stderr, "User has not agreed to TOS."
284           break
285         if e.reason == "AccountDeleted":
286           print >>sys.stderr, "The user account has been deleted."
287           break
288         if e.reason == "AccountDisabled":
289           print >>sys.stderr, "The user account has been disabled."
290           break
291         if e.reason == "ServiceDisabled":
292           print >>sys.stderr, ("The user's access to the service has been "
293                                "disabled.")
294           break
295         if e.reason == "ServiceUnavailable":
296           print >>sys.stderr, "The service is not available; try again later."
297           break
298         raise
299       self._GetAuthCookie(auth_token)
300       return
301 
302   def Send(self, request_path, payload=None,
303            content_type="application/octet-stream",
304            timeout=None,
305            **kwargs):
306     """Sends an RPC and returns the response.
307 
308     Args:
309       request_path: The path to send the request to, eg /api/appversion/create.
310       payload: The body of the request, or None to send an empty request.
311       content_type: The Content-Type header to use.
312       timeout: timeout in seconds; default None i.e. no timeout.
313         (Note: for large requests on OS X, the timeout doesn't work right.)
314       kwargs: Any keyword arguments are converted into query string parameters.
315 
316     Returns:
317       The response body, as a string.
318     """
319     # TODO: Don't require authentication.  Let the server say
320     # whether it is necessary.
321     if not self.authenticated:
322       self._Authenticate()
323 
324     old_timeout = socket.getdefaulttimeout()
325     socket.setdefaulttimeout(timeout)
326     try:
327       tries = 0
328       while True:
329         tries += 1
330         args = dict(kwargs)
331         url = "http://%s%s" % (self.host, request_path)
332         if args:
333           url += "?" + urllib.urlencode(args)
334         req = self._CreateRequest(url=url, data=payload)
335         req.add_header("Content-Type", content_type)
336         try:
337           f = self.opener.open(req)
338           response = f.read()
339           f.close()
340           return response
341         except urllib2.HTTPError, e:
342           if tries > 3:
343             raise
344           elif e.code == 401:
345             self._Authenticate()
346 ##           elif e.code >= 500 and e.code < 600:
347 ##             # Server Error - try again.
348 ##             continue
349           else:
350             raise
351     finally:
352       socket.setdefaulttimeout(old_timeout)
353 
354 
355 class HttpRpcServer(AbstractRpcServer):
356   """Provides a simplified RPC-style interface for HTTP requests."""
357 
358   def _Authenticate(self):
359     """Save the cookie jar after authentication."""
360     super(HttpRpcServer, self)._Authenticate()
361     if self.save_cookies:
362       StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
363       self.cookie_jar.save()
364 
365   def _GetOpener(self):
366     """Returns an OpenerDirector that supports cookies and ignores redirects.
367 
368     Returns:
369       A urllib2.OpenerDirector object.
370     """
371     opener = urllib2.OpenerDirector()
372     opener.add_handler(urllib2.ProxyHandler())
373     opener.add_handler(urllib2.UnknownHandler())
374     opener.add_handler(urllib2.HTTPHandler())
375     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
376     opener.add_handler(urllib2.HTTPSHandler())
377     opener.add_handler(urllib2.HTTPErrorProcessor())
378     if self.save_cookies:
379       self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
380       self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
381       if os.path.exists(self.cookie_file):
382         try:
383           self.cookie_jar.load()
384           self.authenticated = True
385           StatusUpdate("Loaded authentication cookies from %s" %
386                        self.cookie_file)
387         except (cookielib.LoadError, IOError):
388           # Failed to load cookies - just ignore them.
389           pass
390       else:
391         # Create an empty cookie file with mode 600
392         fd = os.open(self.cookie_file, os.O_CREAT, 0600)
393         os.close(fd)
394       # Always chmod the cookie file
395       os.chmod(self.cookie_file, 0600)
396     else:
397       # Don't save cookies across runs of update.py.
398       self.cookie_jar = cookielib.CookieJar()
399     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
400     return opener
401 
402 
403 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
404 parser.add_option("-y", "--assume_yes", action="store_true",
405                   dest="assume_yes", default=False,
406                   help="Assume that the answer to yes/no questions is 'yes'.")
407 # Logging
408 group = parser.add_option_group("Logging options")
409 group.add_option("-q", "--quiet", action="store_const", const=0,
410                  dest="verbose", help="Print errors only.")
411 group.add_option("-v", "--verbose", action="store_const", const=2,
412                  dest="verbose", default=1,
413                  help="Print info level logs (default).")
414 group.add_option("--noisy", action="store_const", const=3,
415                  dest="verbose", help="Print all logs.")
416 # Review server
417 group = parser.add_option_group("Review server options")
418 group.add_option("-s", "--server", action="store", dest="server",
419                  default="codereview.appspot.com",
420                  metavar="SERVER",
421                  help=("The server to upload to. The format is host[:port]. "
422                        "Defaults to '%default'."))
423 group.add_option("-e", "--email", action="store", dest="email",
424                  metavar="EMAIL", default=None,
425                  help="The username to use. Will prompt if omitted.")
426 group.add_option("-H", "--host", action="store", dest="host",
427                  metavar="HOST", default=None,
428                  help="Overrides the Host header sent with all RPCs.")
429 group.add_option("--no_cookies", action="store_false",
430                  dest="save_cookies", default=True,
431                  help="Do not save authentication cookies to local disk.")
432 # Issue
433 group = parser.add_option_group("Issue options")
434 group.add_option("-d", "--description", action="store", dest="description",
435                  metavar="DESCRIPTION", default=None,
436                  help="Optional description when creating an issue.")
437 group.add_option("-f", "--description_file", action="store",
438                  dest="description_file", metavar="DESCRIPTION_FILE",
439                  default=None,
440                  help="Optional path of a file that contains "
441                       "the description when creating an issue.")
442 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
443                  metavar="REVIEWERS", default=None,
444                  help="Add reviewers (comma separated email addresses).")
445 group.add_option("--cc", action="store", dest="cc",
446                  metavar="CC", default=None,
447                  help="Add CC (comma separated email addresses).")
448 # Upload options
449 group = parser.add_option_group("Patch options")
450 group.add_option("-m", "--message", action="store", dest="message",
451                  metavar="MESSAGE", default=None,
452                  help="A message to identify the patch. "
453                       "Will prompt if omitted.")
454 group.add_option("-i", "--issue", type="int", action="store",
455                  metavar="ISSUE", default=None,
456                  help="Issue number to which to add. Defaults to new issue.")
457 group.add_option("--download_base", action="store_true",
458                  dest="download_base", default=False,
459                  help="Base files will be downloaded by the server "
460                  "(side-by-side diffs may not work on files with CRs).")
461 group.add_option("--rev", action="store", dest="revision",
462                  metavar="REV", default=None,
463                  help="Branch/tree/revision to diff against (used by DVCS).")
464 group.add_option("--send_mail", action="store_true",
465                  dest="send_mail", default=False,
466                  help="Send notification email to reviewers.")
467 
468 
469 def GetRpcServer(options):
470   """Returns an instance of an AbstractRpcServer.
471 
472   Returns:
473     A new AbstractRpcServer, on which RPC calls can be made.
474   """
475 
476   rpc_server_class = HttpRpcServer
477 
478   def GetUserCredentials():
479     """Prompts the user for a username and password."""
480     email = options.email
481     if email is None:
482       email = GetEmail("Email (login for uploading to %s)" % options.server)
483     password = getpass.getpass("Password for %s: " % email)
484     return (email, password)
485 
486   # If this is the dev_appserver, use fake authentication.
487   host = (options.host or options.server).lower()
488   if host == "localhost" or host.startswith("localhost:"):
489     email = options.email
490     if email is None:
491       email = "test@example.com"
492       logging.info("Using debug user %s.  Override with --email" % email)
493     server = rpc_server_class(
494         options.server,
495         lambda: (email, "password"),
496         host_override=options.host,
497         extra_headers={"Cookie":
498                        'dev_appserver_login="%s:False"' % email},
499         save_cookies=options.save_cookies)
500     # Don't try to talk to ClientLogin.
501     server.authenticated = True
502     return server
503 
504   return rpc_server_class(options.server, GetUserCredentials,
505                           host_override=options.host,
506                           save_cookies=options.save_cookies)
507 
508 
509 def EncodeMultipartFormData(fields, files):
510   """Encode form fields for multipart/form-data.
511 
512   Args:
513     fields: A sequence of (name, value) elements for regular form fields.
514     files: A sequence of (name, filename, value) elements for data to be
515            uploaded as files.
516   Returns:
517     (content_type, body) ready for httplib.HTTP instance.
518 
519   Source:
520     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
521   """
522   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
523   CRLF = '\r\n'
524   lines = []
525   for (key, value) in fields:
526     lines.append('--' + BOUNDARY)
527     lines.append('Content-Disposition: form-data; name="%s"' % key)
528     lines.append('')
529     lines.append(value)
530   for (key, filename, value) in files:
531     lines.append('--' + BOUNDARY)
532     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
533              (key, filename))
534     lines.append('Content-Type: %s' % GetContentType(filename))
535     lines.append('')
536     lines.append(value)
537   lines.append('--' + BOUNDARY + '--')
538   lines.append('')
539   body = CRLF.join(lines)
540   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
541   return content_type, body
542 
543 
544 def GetContentType(filename):
545   """Helper to guess the content-type from the filename."""
546   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
547 
548 
549 # Use a shell for subcommands on Windows to get a PATH search.
550 use_shell = sys.platform.startswith("win")
551 
552 def RunShellWithReturnCode(command, print_output=False,
553                            universal_newlines=True):
554   """Executes a command and returns the output from stdout and the return code.
555 
556   Args:
557     command: Command to execute.
558     print_output: If True, the output is printed to stdout.
559                   If False, both stdout and stderr are ignored.
560     universal_newlines: Use universal_newlines flag (default: True).
561 
562   Returns:
563     Tuple (output, return code)
564   """
565   logging.info("Running %s", command)
566   p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
567                        shell=use_shell, universal_newlines=universal_newlines)
568   if print_output:
569     output_array = []
570     while True:
571       line = p.stdout.readline()
572       if not line:
573         break
574       print line.strip("\n")
575       output_array.append(line)
576     output = "".join(output_array)
577   else:
578     output = p.stdout.read()
579   p.wait()
580   errout = p.stderr.read()
581   if print_output and errout:
582     print >>sys.stderr, errout
583   p.stdout.close()
584   p.stderr.close()
585   return output, p.returncode
586 
587 
588 def RunShell(command, silent_ok=False, universal_newlines=True,
589              print_output=False):
590   data, retcode = RunShellWithReturnCode(command, print_output,
591                                          universal_newlines)
592   if retcode:
593     ErrorExit("Got error status from %s:\n%s" % (command, data))
594   if not silent_ok and not data:
595     ErrorExit("No output from %s" % command)
596   return data
597 
598 
599 class VersionControlSystem(object):
600   """Abstract base class providing an interface to the VCS."""
601 
602   def __init__(self, options):
603     """Constructor.
604 
605     Args:
606       options: Command line options.
607     """
608     self.options = options
609 
610   def GenerateDiff(self, args):
611     """Return the current diff as a string.
612 
613     Args:
614       args: Extra arguments to pass to the diff command.
615     """
616     raise NotImplementedError(
617         "abstract method -- subclass %s must override" % self.__class__)
618 
619   def GetUnknownFiles(self):
620     """Return a list of files unknown to the VCS."""
621     raise NotImplementedError(
622         "abstract method -- subclass %s must override" % self.__class__)
623 
624   def CheckForUnknownFiles(self):
625     """Show an "are you sure?" prompt if there are unknown files."""
626     unknown_files = self.GetUnknownFiles()
627     if unknown_files:
628       print "The following files are not added to version control:"
629       for line in unknown_files:
630         print line
631       prompt = "Are you sure to continue?(y/N) "
632       answer = raw_input(prompt).strip()
633       if answer != "y":
634         ErrorExit("User aborted")
635 
636   def GetBaseFile(self, filename):
637     """Get the content of the upstream version of a file.
638 
639     Returns:
640       A tuple (base_content, new_content, is_binary, status)
641         base_content: The contents of the base file.
642         new_content: For text files, this is empty.  For binary files, this is
643           the contents of the new file, since the diff output won't contain
644           information to reconstruct the current file.
645         is_binary: True iff the file is binary.
646         status: The status of the file.
647     """
648 
649     raise NotImplementedError(
650         "abstract method -- subclass %s must override" % self.__class__)
651 
652 
653   def GetBaseFiles(self, diff):
654     """Helper that calls GetBase file for each file in the patch.
655 
656     Returns:
657       A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
658       are retrieved based on lines that start with "Index:" or
659       "Property changes on:".
660     """
661     files = {}
662     for line in diff.splitlines(True):
663       if line.startswith('Index:') or line.startswith('Property changes on:'):
664         unused, filename = line.split(':', 1)
665         # On Windows if a file has property changes its filename uses '\'
666         # instead of '/'.
667         filename = filename.strip().replace('\\', '/')
668         files[filename] = self.GetBaseFile(filename)
669     return files
670 
671 
672   def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
673                       files):
674     """Uploads the base files (and if necessary, the current ones as well)."""
675 
676     def UploadFile(filename, file_id, content, is_binary, status, is_base):
677       """Uploads a file to the server."""
678       file_too_large = False
679       if is_base:
680         type = "base"
681       else:
682         type = "current"
683       if len(content) > MAX_UPLOAD_SIZE:
684         print ("Not uploading the %s file for %s because it's too large." %
685                (type, filename))
686         file_too_large = True
687         content = ""
688       checksum = md5(content).hexdigest()
689       if options.verbose > 0 and not file_too_large:
690         print "Uploading %s file for %s" % (type, filename)
691       url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
692       form_fields = [("filename", filename),
693                      ("status", status),
694                      ("checksum", checksum),
695                      ("is_binary", str(is_binary)),
696                      ("is_current", str(not is_base)),
697                     ]
698       if file_too_large:
699         form_fields.append(("file_too_large", "1"))
700       if options.email:
701         form_fields.append(("user", options.email))
702       ctype, body = EncodeMultipartFormData(form_fields,
703                                             [("data", filename, content)])
704       response_body = rpc_server.Send(url, body,
705                                       content_type=ctype)
706       if not response_body.startswith("OK"):
707         StatusUpdate("  --> %s" % response_body)
708         sys.exit(1)
709 
710     patches = dict()
711     [patches.setdefault(v, k) for k, v in patch_list]
712     for filename in patches.keys():
713       base_content, new_content, is_binary, status = files[filename]
714       file_id_str = patches.get(filename)
715       if file_id_str.find("nobase") != -1:
716         base_content = None
717         file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
718       file_id = int(file_id_str)
719       if base_content != None:
720         UploadFile(filename, file_id, base_content, is_binary, status, True)
721       if new_content != None:
722         UploadFile(filename, file_id, new_content, is_binary, status, False)
723 
724   def IsImage(self, filename):
725     """Returns true if the filename has an image extension."""
726     mimetype =  mimetypes.guess_type(filename)[0]
727     if not mimetype:
728       return False
729     return mimetype.startswith("image/")
730 
731   def IsBinary(self, filename):
732     """Returns true if the guessed mimetyped isnt't in text group."""
733     mimetype = mimetypes.guess_type(filename)[0]
734     if not mimetype:
735       return False  # e.g. README, "real" binaries usually have an extension
736     return not mimetype.startswith("text/")
737 
738 
739 class SubversionVCS(VersionControlSystem):
740   """Implementation of the VersionControlSystem interface for Subversion."""
741 
742   def __init__(self, options):
743     super(SubversionVCS, self).__init__(options)
744     if self.options.revision:
745       match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
746       if not match:
747         ErrorExit("Invalid Subversion revision %s." % self.options.revision)
748       self.rev_start = match.group(1)
749       self.rev_end = match.group(3)
750     else:
751       self.rev_start = self.rev_end = None
752     # Cache output from "svn list -r REVNO dirname".
753     # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
754     self.svnls_cache = {}
755     # SVN base URL is required to fetch files deleted in an older revision.
756     # Result is cached to not guess it over and over again in GetBaseFile().
757     required = self.options.download_base or self.options.revision is not None
758     self.svn_base = self._GuessBase(required)
759 
760   def GuessBase(self, required):
761     """Wrapper for _GuessBase."""
762     return self.svn_base
763 
764   def _GuessBase(self, required):
765     """Returns the SVN base URL.
766 
767     Args:
768       required: If true, exits if the url can't be guessed, otherwise None is
769         returned.
770     """
771     info = RunShell(["svn", "info"])
772     for line in info.splitlines():
773       words = line.split()
774       if len(words) == 2 and words[0] == "URL:":
775         url = words[1]
776         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
777         username, netloc = urllib.splituser(netloc)
778         if username:
779           logging.info("Removed username from base URL")
780         if netloc.endswith("svn.python.org"):
781           if netloc == "svn.python.org":
782             if path.startswith("/projects/"):
783               path = path[9:]
784           elif netloc != "pythondev@svn.python.org":
785             ErrorExit("Unrecognized Python URL: %s" % url)
786           base = "http://svn.python.org/view/*checkout*%s/" % path
787           logging.info("Guessed Python base = %s", base)
788         elif netloc.endswith("svn.collab.net"):
789           if path.startswith("/repos/"):
790             path = path[6:]
791           base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
792           logging.info("Guessed CollabNet base = %s", base)
793         elif netloc.endswith(".googlecode.com"):
794           path = path + "/"
795           base = urlparse.urlunparse(("http", netloc, path, params,
796                                       query, fragment))
797           logging.info("Guessed Google Code base = %s", base)
798         else:
799           path = path + "/"
800           base = urlparse.urlunparse((scheme, netloc, path, params,
801                                       query, fragment))
802           logging.info("Guessed base = %s", base)
803         return base
804     if required:
805       ErrorExit("Can't find URL in output from svn info")
806     return None
807 
808   def GenerateDiff(self, args):
809     cmd = ["svn", "diff"]
810     if self.options.revision:
811       cmd += ["-r", self.options.revision]
812     cmd.extend(args)
813     data = RunShell(cmd)
814     count = 0
815     for line in data.splitlines():
816       if line.startswith("Index:") or line.startswith("Property changes on:"):
817         count += 1
818         logging.info(line)
819     if not count:
820       ErrorExit("No valid patches found in output from svn diff")
821     return data
822 
823   def _CollapseKeywords(self, content, keyword_str):
824     """Collapses SVN keywords."""
825     # svn cat translates keywords but svn diff doesn't. As a result of this
826     # behavior patching.PatchChunks() fails with a chunk mismatch error.
827     # This part was originally written by the Review Board development team
828     # who had the same problem (http://reviews.review-board.org/r/276/).
829     # Mapping of keywords to known aliases
830     svn_keywords = {
831       # Standard keywords
832       'Date':                ['Date', 'LastChangedDate'],
833       'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],
834       'Author':              ['Author', 'LastChangedBy'],
835       'HeadURL':             ['HeadURL', 'URL'],
836       'Id':                  ['Id'],
837 
838       # Aliases
839       'LastChangedDate':     ['LastChangedDate', 'Date'],
840       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
841       'LastChangedBy':       ['LastChangedBy', 'Author'],
842       'URL':                 ['URL', 'HeadURL'],
843     }
844 
845     def repl(m):
846        if m.group(2):
847          return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
848        return "$%s$" % m.group(1)
849     keywords = [keyword
850                 for name in keyword_str.split(" ")
851                 for keyword in svn_keywords.get(name, [])]
852     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
853 
854   def GetUnknownFiles(self):
855     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
856     unknown_files = []
857     for line in status.split("\n"):
858       if line and line[0] == "?":
859         unknown_files.append(line)
860     return unknown_files
861 
862   def ReadFile(self, filename):
863     """Returns the contents of a file."""
864     file = open(filename, 'rb')
865     result = ""
866     try:
867       result = file.read()
868     finally:
869       file.close()
870     return result
871 
872   def GetStatus(self, filename):
873     """Returns the status of a file."""
874     if not self.options.revision:
875       status = RunShell(["svn", "status", "--ignore-externals", filename])
876       if not status:
877         ErrorExit("svn status returned no output for %s" % filename)
878       status_lines = status.splitlines()
879       # If file is in a cl, the output will begin with
880       # "\n--- Changelist 'cl_name':\n".  See
881       # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
882       if (len(status_lines) == 3 and
883           not status_lines[0] and
884           status_lines[1].startswith("--- Changelist")):
885         status = status_lines[2]
886       else:
887         status = status_lines[0]
888     # If we have a revision to diff against we need to run "svn list"
889     # for the old and the new revision and compare the results to get
890     # the correct status for a file.
891     else:
892       dirname, relfilename = os.path.split(filename)
893       if dirname not in self.svnls_cache:
894         cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
895         out, returncode = RunShellWithReturnCode(cmd)
896         if returncode:
897           ErrorExit("Failed to get status for %s." % filename)
898         old_files = out.splitlines()
899         args = ["svn", "list"]
900         if self.rev_end:
901           args += ["-r", self.rev_end]
902         cmd = args + [dirname or "."]
903         out, returncode = RunShellWithReturnCode(cmd)
904         if returncode:
905           ErrorExit("Failed to run command %s" % cmd)
906         self.svnls_cache[dirname] = (old_files, out.splitlines())
907       old_files, new_files = self.svnls_cache[dirname]
908       if relfilename in old_files and relfilename not in new_files:
909         status = "D   "
910       elif relfilename in old_files and relfilename in new_files:
911         status = "M   "
912       else:
913         status = "A   "
914     return status
915 
916   def GetBaseFile(self, filename):
917     status = self.GetStatus(filename)
918     base_content = None
919     new_content = None
920 
921     # If a file is copied its status will be "A  +", which signifies
922     # "addition-with-history".  See "svn st" for more information.  We need to
923     # upload the original file or else diff parsing will fail if the file was
924     # edited.
925     if status[0] == "A" and status[3] != "+":
926       # We'll need to upload the new content if we're adding a binary file
927       # since diff's output won't contain it.
928       mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
929                           silent_ok=True)
930       base_content = ""
931       is_binary = bool(mimetype) and not mimetype.startswith("text/")
932       if is_binary and self.IsImage(filename):
933         new_content = self.ReadFile(filename)
934     elif (status[0] in ("M", "D", "R") or
935           (status[0] == "A" and status[3] == "+") or  # Copied file.
936           (status[0] == " " and status[1] == "M")):  # Property change.
937       args = []
938       if self.options.revision:
939         url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
940       else:
941         # Don't change filename, it's needed later.
942         url = filename
943         args += ["-r", "BASE"]
944       cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
945       mimetype, returncode = RunShellWithReturnCode(cmd)
946       if returncode:
947         # File does not exist in the requested revision.
948         # Reset mimetype, it contains an error message.
949         mimetype = ""
950       get_base = False
951       is_binary = bool(mimetype) and not mimetype.startswith("text/")
952       if status[0] == " ":
953         # Empty base content just to force an upload.
954         base_content = ""
955       elif is_binary:
956         if self.IsImage(filename):
957           get_base = True
958           if status[0] == "M":
959             if not self.rev_end:
960               new_content = self.ReadFile(filename)
961             else:
962               url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
963               new_content = RunShell(["svn", "cat", url],
964                                      universal_newlines=True, silent_ok=True)
965         else:
966           base_content = ""
967       else:
968         get_base = True
969 
970       if get_base:
971         if is_binary:
972           universal_newlines = False
973         else:
974           universal_newlines = True
975         if self.rev_start:
976           # "svn cat -r REV delete_file.txt" doesn't work. cat requires
977           # the full URL with "@REV" appended instead of using "-r" option.
978           url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
979           base_content = RunShell(["svn", "cat", url],
980                                   universal_newlines=universal_newlines,
981                                   silent_ok=True)
982         else:
983           base_content = RunShell(["svn", "cat", filename],
984                                   universal_newlines=universal_newlines,
985                                   silent_ok=True)
986         if not is_binary:
987           args = []
988           if self.rev_start:
989             url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
990           else:
991             url = filename
992             args += ["-r", "BASE"]
993           cmd = ["svn"] + args + ["propget", "svn:keywords", url]
994           keywords, returncode = RunShellWithReturnCode(cmd)
995           if keywords and not returncode:
996             base_content = self._CollapseKeywords(base_content, keywords)
997     else:
998       StatusUpdate("svn status returned unexpected output: %s" % status)
999       sys.exit(1)
1000     return base_content, new_content, is_binary, status[0:5]
1001 
1002 
1003 class GitVCS(VersionControlSystem):
1004   """Implementation of the VersionControlSystem interface for Git."""
1005 
1006   NULL_HASH = "0"*40
1007 
1008   def __init__(self, options):
1009     super(GitVCS, self).__init__(options)
1010     # Map of filename -> (hash before, hash after) of base file.
1011     self.base_hashes = {}
1012 
1013   def GenerateDiff(self, extra_args):
1014     # This is more complicated than svn's GenerateDiff because we must convert
1015     # the diff output to include an svn-style "Index:" line as well as record
1016     # the hashes of the base files, so we can upload them along with our diff.
1017     if self.options.revision:
1018       extra_args = [self.options.revision] + extra_args
1019     gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)
1020     svndiff = []
1021     filecount = 0
1022     filename = None
1023     for line in gitdiff.splitlines():
1024       match = re.match(r"diff --git a/(.*) b/.*$", line)
1025       if match:
1026         filecount += 1
1027         filename = match.group(1)
1028         svndiff.append("Index: %s\n" % filename)
1029       else:
1030         # The "index" line in a git diff looks like this (long hashes elided):
1031         #   index 82c0d44..b2cee3f 100755
1032         # We want to save the left hash, as that identifies the base file.
1033         match = re.match(r"index (\w+)\.\.(\w+)", line)
1034         if match:
1035           self.base_hashes[filename] = (match.group(1), match.group(2))
1036       svndiff.append(line + "\n")
1037     if not filecount:
1038       ErrorExit("No valid patches found in output from git diff")
1039     return "".join(svndiff)
1040 
1041   def GetUnknownFiles(self):
1042     status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
1043                       silent_ok=True)
1044     return status.splitlines()
1045 
1046   def GetFileContent(self, file_hash, is_binary):
1047     """Returns the content of a file identified by its git hash."""
1048     data, retcode = RunShellWithReturnCode(["git", "show", file_hash],
1049                                             universal_newlines=not is_binary)
1050     if retcode:
1051       ErrorExit("Got error status from 'git show %s'" % file_hash)
1052     return data
1053 
1054   def GetBaseFile(self, filename):
1055     hash_before, hash_after = self.base_hashes[filename]
1056     base_content = None
1057     new_content = None
1058     is_binary = self.IsBinary(filename)
1059 
1060     if hash_before == self.NULL_HASH:  # All-zero hash indicates no base file.
1061       status = "A"
1062       base_content = ""
1063     else:
1064       status = "M"
1065       if not is_binary or self.IsImage(filename):
1066         base_content = self.GetFileContent(hash_before, is_binary)
1067 
1068     if is_binary and self.IsImage(filename) and not hash_after == "0" * 40:
1069       new_content = self.GetFileContent(hash_after, is_binary)
1070 
1071     if hash_after == self.NULL_HASH:
1072       status = "D"
1073     return (base_content, new_content, is_binary, status)
1074 
1075 
1076 class MercurialVCS(VersionControlSystem):
1077   """Implementation of the VersionControlSystem interface for Mercurial."""
1078 
1079   def __init__(self, options, repo_dir):
1080     super(MercurialVCS, self).__init__(options)
1081     # Absolute path to repository (we can be in a subdir)
1082     self.repo_dir = os.path.normpath(repo_dir)
1083     # Compute the subdir
1084     cwd = os.path.normpath(os.getcwd())
1085     assert cwd.startswith(self.repo_dir)
1086     self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
1087     if self.options.revision:
1088       self.base_rev = self.options.revision
1089     else:
1090       self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
1091 
1092   def _GetRelPath(self, filename):
1093     """Get relative path of a file according to the current directory,
1094     given its logical path in the repo."""
1095     assert filename.startswith(self.subdir), filename
1096     return filename[len(self.subdir):].lstrip(r"\/")
1097 
1098   def GenerateDiff(self, extra_args):
1099     # If no file specified, restrict to the current subdir
1100     extra_args = extra_args or ["."]
1101     cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
1102     data = RunShell(cmd, silent_ok=True)
1103     svndiff = []
1104     filecount = 0
1105     for line in data.splitlines():
1106       m = re.match("diff --git a/(\S+) b/(\S+)", line)
1107       if m:
1108         # Modify line to make it look like as it comes from svn diff.
1109         # With this modification no changes on the server side are required
1110         # to make upload.py work with Mercurial repos.
1111         # NOTE: for proper handling of moved/copied files, we have to use
1112         # the second filename.
1113         filename = m.group(2)
1114         svndiff.append("Index: %s" % filename)
1115         svndiff.append("=" * 67)
1116         filecount += 1
1117         logging.info(line)
1118       else:
1119         svndiff.append(line)
1120     if not filecount:
1121       ErrorExit("No valid patches found in output from hg diff")
1122     return "\n".join(svndiff) + "\n"
1123 
1124   def GetUnknownFiles(self):
1125     """Return a list of files unknown to the VCS."""
1126     args = []
1127     status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
1128         silent_ok=True)
1129     unknown_files = []
1130     for line in status.splitlines():
1131       st, fn = line.split(" ", 1)
1132       if st == "?":
1133         unknown_files.append(fn)
1134     return unknown_files
1135 
1136   def GetBaseFile(self, filename):
1137     # "hg status" and "hg cat" both take a path relative to the current subdir
1138     # rather than to the repo root, but "hg diff" has given us the full path
1139     # to the repo root.
1140     base_content = ""
1141     new_content = None
1142     is_binary = False
1143     oldrelpath = relpath = self._GetRelPath(filename)
1144     # "hg status -C" returns two lines for moved/copied files, one otherwise
1145     out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
1146     out = out.splitlines()
1147     # HACK: strip error message about missing file/directory if it isn't in
1148     # the working copy
1149     if out[0].startswith('%s: ' % relpath):
1150       out = out[1:]
1151     if len(out) > 1:
1152       # Moved/copied => considered as modified, use old filename to
1153       # retrieve base contents
1154       oldrelpath = out[1].strip()
1155       status = "M"
1156     else:
1157       status, _ = out[0].split(' ', 1)
1158     if ":" in self.base_rev:
1159       base_rev = self.base_rev.split(":", 1)[0]
1160     else:
1161       base_rev = self.base_rev
1162     if status != "A":
1163       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
1164         silent_ok=True)
1165       is_binary = "\0" in base_content  # Mercurial's heuristic
1166     if status != "R":
1167       new_content = open(relpath, "rb").read()
1168       is_binary = is_binary or "\0" in new_content
1169     if is_binary and base_content:
1170       # Fetch again without converting newlines
1171       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
1172         silent_ok=True, universal_newlines=False)
1173     if not is_binary or not self.IsImage(relpath):
1174       new_content = None
1175     return base_content, new_content, is_binary, status
1176 
1177 
1178 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
1179 def SplitPatch(data):
1180   """Splits a patch into separate pieces for each file.
1181 
1182   Args:
1183     data: A string containing the output of svn diff.
1184 
1185   Returns:
1186     A list of 2-tuple (filename, text) where text is the svn diff output
1187       pertaining to filename.
1188   """
1189   patches = []
1190   filename = None
1191   diff = []
1192   for line in data.splitlines(True):
1193     new_filename = None
1194     if line.startswith('Index:'):
1195       unused, new_filename = line.split(':', 1)
1196       new_filename = new_filename.strip()
1197     elif line.startswith('Property changes on:'):
1198       unused, temp_filename = line.split(':', 1)
1199       # When a file is modified, paths use '/' between directories, however
1200       # when a property is modified '\' is used on Windows.  Make them the same
1201       # otherwise the file shows up twice.
1202       temp_filename = temp_filename.strip().replace('\\', '/')
1203       if temp_filename != filename:
1204         # File has property changes but no modifications, create a new diff.
1205         new_filename = temp_filename
1206     if new_filename:
1207       if filename and diff:
1208         patches.append((filename, ''.join(diff)))
1209       filename = new_filename
1210       diff = [line]
1211       continue
1212     if diff is not None:
1213       diff.append(line)
1214   if filename and diff:
1215     patches.append((filename, ''.join(diff)))
1216   return patches
1217 
1218 
1219 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
1220   """Uploads a separate patch for each file in the diff output.
1221 
1222   Returns a list of [patch_key, filename] for each file.
1223   """
1224   patches = SplitPatch(data)
1225   rv = []
1226   for patch in patches:
1227     if len(patch[1]) > MAX_UPLOAD_SIZE:
1228       print ("Not uploading the patch for " + patch[0] +
1229              " because the file is too large.")
1230       continue
1231     form_fields = [("filename", patch[0])]
1232     if not options.download_base:
1233       form_fields.append(("content_upload", "1"))
1234     files = [("data", "data.diff", patch[1])]
1235     ctype, body = EncodeMultipartFormData(form_fields, files)
1236     url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
1237     print "Uploading patch for " + patch[0]
1238     response_body = rpc_server.Send(url, body, content_type=ctype)
1239     lines = response_body.splitlines()
1240     if not lines or lines[0] != "OK":
1241       StatusUpdate("  --> %s" % response_body)
1242       sys.exit(1)
1243     rv.append([lines[1], patch[0]])
1244   return rv
1245 
1246 
1247 def GuessVCSName():
1248   """Helper to guess the version control system.
1249 
1250   This examines the current directory, guesses which VersionControlSystem
1251   we're using, and returns an string indicating which VCS is detected.
1252 
1253   Returns:
1254     A pair (vcs, output).  vcs is a string indicating which VCS was detected
1255     and is one of VCS_GIT, VCS_MERCURIAL, VCS_SUBVERSION, or VCS_UNKNOWN.
1256     output is a string containing any interesting output from the vcs
1257     detection routine, or None if there is nothing interesting.
1258   """
1259   # Mercurial has a command to get the base directory of a repository
1260   # Try running it, but don't die if we don't have hg installed.
1261   # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
1262   try:
1263     out, returncode = RunShellWithReturnCode(["hg", "root"])
1264     if returncode == 0:
1265       return (VCS_MERCURIAL, out.strip())
1266   except OSError, (errno, message):
1267     if errno != 2:  # ENOENT -- they don't have hg installed.
1268       raise
1269 
1270   # Subversion has a .svn in all working directories.
1271   if os.path.isdir('.svn'):
1272     logging.info("Guessed VCS = Subversion")
1273     return (VCS_SUBVERSION, None)
1274 
1275   # Git has a command to test if you're in a git tree.
1276   # Try running it, but don't die if we don't have git installed.
1277   try:
1278     out, returncode = RunShellWithReturnCode(["git", "rev-parse",
1279                                               "--is-inside-work-tree"])
1280     if returncode == 0:
1281       return (VCS_GIT, None)
1282   except OSError, (errno, message):
1283     if errno != 2:  # ENOENT -- they don't have git installed.
1284       raise
1285 
1286   return (VCS_UNKNOWN, None)
1287 
1288 
1289 def GuessVCS(options):
1290   """Helper to guess the version control system.
1291 
1292   This examines the current directory, guesses which VersionControlSystem
1293   we're using, and returns an instance of the appropriate class.  Exit with an
1294   error if we can't figure it out.
1295 
1296   Returns:
1297     A VersionControlSystem instance. Exits if the VCS can't be guessed.
1298   """
1299   (vcs, extra_output) = GuessVCSName()
1300   if vcs == VCS_MERCURIAL:
1301     return MercurialVCS(options, extra_output)
1302   elif vcs == VCS_SUBVERSION:
1303     return SubversionVCS(options)
1304   elif vcs == VCS_GIT:
1305     return GitVCS(options)
1306 
1307   ErrorExit(("Could not guess version control system. "
1308              "Are you in a working copy directory?"))
1309 
1310 
1311 def RealMain(argv, data=None):
1312   """The real main function.
1313 
1314   Args:
1315     argv: Command line arguments.
1316     data: Diff contents. If None (default) the diff is generated by
1317       the VersionControlSystem implementation returned by GuessVCS().
1318 
1319   Returns:
1320     A 2-tuple (issue id, patchset id).
1321     The patchset id is None if the base files are not uploaded by this
1322     script (applies only to SVN checkouts).
1323   """
1324   logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
1325                               "%(lineno)s %(message)s "))
1326   os.environ['LC_ALL'] = 'C'
1327   options, args = parser.parse_args(argv[1:])
1328   global verbosity
1329   verbosity = options.verbose
1330   if verbosity >= 3:
1331     logging.getLogger().setLevel(logging.DEBUG)
1332   elif verbosity >= 2:
1333     logging.getLogger().setLevel(logging.INFO)
1334   vcs = GuessVCS(options)
1335   if isinstance(vcs, SubversionVCS):
1336     # base field is only allowed for Subversion.
1337     # Note: Fetching base files may become deprecated in future releases.
1338     base = vcs.GuessBase(options.download_base)
1339   else:
1340     base = None
1341   if not base and options.download_base:
1342     options.download_base = True
1343     logging.info("Enabled upload of base file")
1344   if not options.assume_yes:
1345     vcs.CheckForUnknownFiles()
1346   if data is None:
1347     data = vcs.GenerateDiff(args)
1348   files = vcs.GetBaseFiles(data)
1349   if verbosity >= 1:
1350     print "Upload server:", options.server, "(change with -s/--server)"
1351   if options.issue:
1352     prompt = "Message describing this patch set: "
1353   else:
1354     prompt = "New issue subject: "
1355   message = options.message or raw_input(prompt).strip()
1356   if not message:
1357     ErrorExit("A non-empty message is required")
1358   rpc_server = GetRpcServer(options)
1359   form_fields = [("subject", message)]
1360   if base:
1361     form_fields.append(("base", base))
1362   if options.issue:
1363     form_fields.append(("issue", str(options.issue)))
1364   if options.email:
1365     form_fields.append(("user", options.email))
1366   if options.reviewers:
1367     for reviewer in options.reviewers.split(','):
1368       if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
1369         ErrorExit("Invalid email address: %s" % reviewer)
1370     form_fields.append(("reviewers", options.reviewers))
1371   if options.cc:
1372     for cc in options.cc.split(','):
1373       if "@" in cc and not cc.split("@")[1].count(".") == 1:
1374         ErrorExit("Invalid email address: %s" % cc)
1375     form_fields.append(("cc", options.cc))
1376   description = options.description
1377   if options.description_file:
1378     if options.description:
1379       ErrorExit("Can't specify description and description_file")
1380     file = open(options.description_file, 'r')
1381     description = file.read()
1382     file.close()
1383   if description:
1384     form_fields.append(("description", description))
1385   # Send a hash of all the base file so the server can determine if a copy
1386   # already exists in an earlier patchset.
1387   base_hashes = ""
1388   for file, info in files.iteritems():
1389     if not info[0] is None:
1390       checksum = md5(info[0]).hexdigest()
1391       if base_hashes:
1392         base_hashes += "|"
1393       base_hashes += checksum + ":" + file
1394   form_fields.append(("base_hashes", base_hashes))
1395   # If we're uploading base files, don't send the email before the uploads, so
1396   # that it contains the file status.
1397   if options.send_mail and options.download_base:
1398     form_fields.append(("send_mail", "1"))
1399   if not options.download_base:
1400     form_fields.append(("content_upload", "1"))
1401   if len(data) > MAX_UPLOAD_SIZE:
1402     print "Patch is large, so uploading file patches separately."
1403     uploaded_diff_file = []
1404     form_fields.append(("separate_patches", "1"))
1405   else:
1406     uploaded_diff_file = [("data", "data.diff", data)]
1407   ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
1408   response_body = rpc_server.Send("/upload", body, content_type=ctype)
1409   patchset = None
1410   if not options.download_base or not uploaded_diff_file:
1411     lines = response_body.splitlines()
1412     if len(lines) >= 2:
1413       msg = lines[0]
1414       patchset = lines[1].strip()
1415       patches = [x.split(" ", 1) for x in lines[2:]]
1416     else:
1417       msg = response_body
1418   else:
1419     msg = response_body
1420   StatusUpdate(msg)
1421   if not response_body.startswith("Issue created.") and \
1422   not response_body.startswith("Issue updated."):
1423     sys.exit(0)
1424   issue = msg[msg.rfind("/")+1:]
1425 
1426   if not uploaded_diff_file:
1427     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
1428     if not options.download_base:
1429       patches = result
1430 
1431   if not options.download_base:
1432     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
1433     if options.send_mail:
1434       rpc_server.Send("/" + issue + "/mail", payload="")
1435   return issue, patchset
1436 
1437 
1438 def main():
1439   try:
1440     RealMain(sys.argv)
1441   except KeyboardInterrupt:
1442     print
1443     StatusUpdate("Interrupted.")
1444     sys.exit(1)
1445 
1446 
1447 if __name__ == "__main__":
1448   main()