commit 469dd7264e07bc434adc24b250cd154db9e18d02 from: Stefan Sperling date: Fri Mar 20 11:44:14 2020 UTC add support for repository mirrors to 'got clone' and 'got fetch' commit - b364b1c2040ad877ea7534f23a740602944350b8 commit + 469dd7264e07bc434adc24b250cd154db9e18d02 blob - ec8f587e42bb8d678f45c8bd23aa2c37f49180ae blob + 81008498da32f07c86f4e614d70cc4e5a73f817a --- got/got.1 +++ got/got.1 @@ -190,6 +190,35 @@ The options for .Cm got clone are as follows: .Bl -tag -width Ds +.It Fl m +Create the cloned repository as a mirror of the original repository. +This is useful if the cloned repository will not be used to store +local changes as created by +.Cm got commit . +.Pp +The repository's +.Pa config +file will be set up with the +.Dq mirror +option enabled, such that +.Cm got fetch +or +.Xr git-fetch 1 +will write incoming changes directly to branches in the +.Dq refs/heads/ +reference namespace, rather than to branches in the +.Dq refs/remotes/ +namespace. +This avoids the usual requirement of having to run +.Cm got rebase +after +.Cm got fetch +in order to make incoming changes appear on branches in the +.Dq refs/heads/ +namespace. +But maintaining custom branches with local changes in the cloned +repository becomes difficult since local changes are at risk of +being discarded whenever incoming changes are fetched. .It Fl q Suppress progress reporting output. The same option will be passed to @@ -220,16 +249,26 @@ file of the repository, as created by .Pp Branch references in the .Dq refs/remotes/ -reference namespace will be updated to point at the newly fetched commits. -The +reference namespace will be updated to point at the newly fetched commits, +and the .Cm got rebase command can then be used to make new changes visible on branches in the .Dq refs/heads/ -reference namespace. +reference namespace, merging incoming changes with local changes as necessary. .Pp -Existing references in the +However, if the repository is configured as a mirror then all references will +be updated as needed to match the corresponding references in the remote +repository, including branches in the +.Dq refs/heads/ +reference namespace. +If those branches contained local commits, these will no longer be reachable +via a reference and will therefore be at risk of being discarded by Git's +garbage collector. +.Pp +In any case, existing references in the .Dq refs/tags/ -namespace will be changed to match tags contained in the remote repository. +namespace will always be changed to match tags contained in the remote +repository. .Pp The options for .Cm got fetch @@ -1715,13 +1754,32 @@ to be amended and perhaps committed again: .Dl $ # now back out the previous backout :-) .Dl $ got backout unified-buffer-cache .Pp -Fetch new upstream commits into the local repository's master branch. -This step currently requires -.Xr git 1 : +Fetch new upstream commits into the local repository's +.Dq origin/master +branch: .Pp +.Dl $ cd /usr/src +.Dl $ got fetch +.Pp +In a repository created with a HTTP URL and +.Cm git clone --bare +the +.Xr git-fetch 1 +command must be used instead: +.Pp .Dl $ cd /var/git/src.git -.Dl $ git fetch origin master:master +.Dl $ git fetch origin master:refs/remotes/origin/master .Pp +Rebase the local +.Dq master +branch to merge the new changes that are now visible on the +.Dq origin/master +branch: +.Pp +.Dl $ cd /usr/src +.Dl $ got update -b origin/master +.Dl $ got rebase master +.Pp Rebase the .Dq unified-buffer-cache branch on top of the new head commit of the @@ -1746,11 +1804,12 @@ branch: .Dl $ got update -c master .Dl $ got histedit .Pp -Additional steps are necessary if local changes need to be pushed back +.Pp +Additional steps may be necessary if local changes need to be pushed back to the remote repository, which currently requires .Cm git push . Before working against existing branches in a repository cloned with -.Dq git clone --bare +.Cm git clone --bare instead of .Cm got clone , a Git @@ -1769,9 +1828,15 @@ command: .Pp .Dl $ cd /var/git/repo .Dl $ git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' +Additionally, the +.Dq mirror +option must be disabled: .Pp +.Dl $ cd /var/git/repo +.Dl $ git config remote.origin.mirror false +.Pp Alternatively, the following -.Pa fetch +.Xr git-fetch 1 configuration item can be added manually to the Git repository's .Pa config file: @@ -1779,6 +1844,7 @@ file: .Dl [remote "origin"] .Dl url = ... .Dl fetch = +refs/heads/*:refs/remotes/origin/* +.Dl mirror = false .Pp This configuration leaves the local repository's .Dq refs/heads @@ -1789,12 +1855,15 @@ and, if needed, created with .Pp Branches in the .Dq remotes/origin -namespace can be updated with incoming changes from the remote +namespace can now be updated with incoming changes from the remote repository with -.Cm got fetch : +.Cm got fetch +or +.Xr git-fetch 1 +without extra command line arguments: .Pp .Dl $ cd /var/git/repo -.Dl $ got fetch +.Dl $ git fetch .Pp To make changes fetched from the remote repository appear on the .Dq master blob - 5ab89fae5b5401718f66d0ab3657a026498e98a6 blob + 740999cee9b25b1a0f944444b86edbe1b46e26d3 --- got/got.c +++ got/got.c @@ -811,7 +811,7 @@ done: __dead static void usage_clone(void) { - fprintf(stderr, "usage: %s clone [-q] [-v] repository-url " + fprintf(stderr, "usage: %s clone [-m] [-q] [-v] repository-url " "[target-directory]\n", getprogname()); exit(1); } @@ -903,13 +903,16 @@ cmd_clone(int argc, char *argv[]) char *gitconfig = NULL; FILE *gitconfig_file = NULL; ssize_t n; - int verbosity = 0; + int verbosity = 0, mirror_references = 0; TAILQ_INIT(&refs); TAILQ_INIT(&symrefs); - while ((ch = getopt(argc, argv, "vq")) != -1) { + while ((ch = getopt(argc, argv, "mvq")) != -1) { switch (ch) { + case 'm': + mirror_references = 1; + break; case 'v': if (verbosity < 0) verbosity = 0; @@ -1022,14 +1025,25 @@ cmd_clone(int argc, char *argv[]) error = got_error_from_errno2("fopen", gitconfig_path); goto done; } - if (asprintf(&gitconfig, - "[remote \"%s\"]\n" - "\turl = %s\n" - "\tfetch = +refs/heads/*:refs/remotes/%s/*\n", - GOT_FETCH_DEFAULT_REMOTE_NAME, git_url, - GOT_FETCH_DEFAULT_REMOTE_NAME) == -1) { - error = got_error_from_errno("asprintf"); - goto done; + if (mirror_references) { + if (asprintf(&gitconfig, + "[remote \"%s\"]\n" + "\turl = %s\n" + "\tmirror = true\n", + GOT_FETCH_DEFAULT_REMOTE_NAME, git_url) == -1) { + error = got_error_from_errno("asprintf"); + goto done; + } + } else { + if (asprintf(&gitconfig, + "[remote \"%s\"]\n" + "\turl = %s\n" + "\tfetch = +refs/heads/*:refs/remotes/%s/*\n", + GOT_FETCH_DEFAULT_REMOTE_NAME, git_url, + GOT_FETCH_DEFAULT_REMOTE_NAME) == -1) { + error = got_error_from_errno("asprintf"); + goto done; + } } n = fwrite(gitconfig, 1, strlen(gitconfig), gitconfig_file); if (n != strlen(gitconfig)) { @@ -1042,8 +1056,8 @@ cmd_clone(int argc, char *argv[]) fpa.last_p_resolved = -1; fpa.verbosity = verbosity; error = got_fetch_pack(&pack_hash, &refs, &symrefs, - GOT_FETCH_DEFAULT_REMOTE_NAME, fetchfd, repo, - fetch_progress, &fpa); + GOT_FETCH_DEFAULT_REMOTE_NAME, mirror_references, + fetchfd, repo, fetch_progress, &fpa); if (error) goto done; @@ -1069,6 +1083,9 @@ cmd_clone(int argc, char *argv[]) if (error) goto done; + if (mirror_references) + continue; + if (strncmp("refs/heads/", refname, 11) != 0) continue; @@ -1118,7 +1135,8 @@ cmd_clone(int argc, char *argv[]) } if (verbosity >= 0) - printf("Created cloned repository '%s'\n", repo_path); + printf("Created %s repository '%s'\n", + mirror_references ? "mirrored" : "cloned", repo_path); done: if (fetchfd != -1 && close(fetchfd) == -1 && error == NULL) error = got_error_from_errno("close"); @@ -1372,7 +1390,7 @@ cmd_fetch(int argc, char *argv[]) fpa.last_p_resolved = -1; fpa.verbosity = verbosity; error = got_fetch_pack(&pack_hash, &refs, &symrefs, remote->name, - fetchfd, repo, fetch_progress, &fpa); + remote->mirror_references, fetchfd, repo, fetch_progress, &fpa); if (error) goto done; blob - 43417f24df05740c8f026cd7b896c75491e9afc4 blob + 9df27fa28b21dbc3fd4faab5673e0efdd0eaa24c --- include/got_fetch.h +++ include/got_fetch.h @@ -61,4 +61,4 @@ typedef const struct got_error *(*got_fetch_progress_c */ const struct got_error *got_fetch_pack(struct got_object_id **, struct got_pathlist_head *, struct got_pathlist_head *, const char *, - int, struct got_repository *, got_fetch_progress_cb, void *); + int, int, struct got_repository *, got_fetch_progress_cb, void *); blob - 106390c3755823cde7aec38e7f08f87dd09e1afc blob + 1e0ca3b4ba3d818cceefe6c470559274e954788c --- include/got_repository.h +++ include/got_repository.h @@ -51,6 +51,12 @@ const char *got_repo_get_gitconfig_owner(struct got_re struct got_remote_repo { char *name; char *url; + + /* + * If set, references are mirrored 1:1 into the local repository. + * If not set, references are mapped into "refs/remotes/$name/". + */ + int mirror_references; }; /* Obtain the list of remote repositories parsed from gitconfig. */ blob - 132da7a4423d03eb7ad5f8bd359050d9841a9cc6 blob + 8edac2b231e81f398c49e2d59c055e3c84a443b1 --- lib/fetch.c +++ lib/fetch.c @@ -387,9 +387,9 @@ check_pack_hash(int fd, size_t sz, uint8_t *hcomp) const struct got_error* got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs, - struct got_pathlist_head *symrefs, const char *remote_name, int fetchfd, - struct got_repository *repo, got_fetch_progress_cb progress_cb, - void *progress_arg) + struct got_pathlist_head *symrefs, const char *remote_name, + int mirror_references, int fetchfd, struct got_repository *repo, + got_fetch_progress_cb progress_cb, void *progress_arg) { int imsg_fetchfds[2], imsg_idxfds[2]; int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1; @@ -418,9 +418,12 @@ got_fetch_pack(struct got_object_id **pack_hash, struc TAILQ_INIT(&have_refs); SIMPLEQ_INIT(&my_refs); - if (asprintf(&ref_prefix, "refs/remotes/%s/", remote_name) == -1) - return got_error_from_errno("asprintf"); - ref_prefixlen = strlen(ref_prefix); + if (!mirror_references) { + if (asprintf(&ref_prefix, "refs/remotes/%s/", + remote_name) == -1) + return got_error_from_errno("asprintf"); + ref_prefixlen = strlen(ref_prefix); + } err = got_ref_list(&my_refs, repo, NULL, got_ref_cmp_by_name, NULL); if (err) @@ -434,6 +437,23 @@ got_fetch_pack(struct got_object_id **pack_hash, struc continue; refname = got_ref_get_name(re->ref); + + if (mirror_references) { + char *name; + err = got_ref_resolve(&id, repo, re->ref); + if (err) + goto done; + name = strdup(refname); + if (name == NULL) { + err = got_error_from_errno("strdup"); + goto done; + } + err = got_pathlist_append(&have_refs, name, id); + if (err) + goto done; + continue; + } + if (strncmp("refs/tags/", refname, 10) == 0) { char *tagname; blob - 11db131869f793fff0ce97b621c4c2b7624386b5 blob + 5d6fc07ab4644eda2d4e44d188b854367410fd15 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -341,6 +341,7 @@ struct got_imsg_traversed_commits { struct got_imsg_remote { size_t name_len; size_t url_len; + int mirror_references; /* Followed by name_len + url_len data bytes. */ }; blob - d869c213630f03ce2acc89b69d0b849c0d947184 blob + e0f404308ac697a06f62bd1ca78426720a294b81 --- lib/privsep.c +++ lib/privsep.c @@ -1930,6 +1930,13 @@ got_privsep_send_gitconfig_remotes(struct imsgbuf *ibu return err; } if (imsg_add(wbuf, remotes[i].url, iremote.url_len) == -1) { + err = got_error_from_errno( + "imsg_add GITCONFIG_REMOTE"); + ibuf_free(wbuf); + return err; + } + if (imsg_add(wbuf, &remotes[i].mirror_references, + sizeof(iremote.mirror_references)) == -1) { err = got_error_from_errno( "imsg_add GITCONFIG_REMOTE"); ibuf_free(wbuf); @@ -2023,6 +2030,7 @@ got_privsep_recv_gitconfig_remotes(struct got_remote_r free(remote->name); break; } + remote->mirror_references = iremote.mirror_references; (*nremotes)++; break; default: blob - 4d1f4a8e9ef496a4c7a895a9789cdf0ba1c63979 blob + 6f203d17788fd8772211527b029238c7be2d3e99 --- libexec/got-read-gitconfig/got-read-gitconfig.c +++ libexec/got-read-gitconfig/got-read-gitconfig.c @@ -108,7 +108,7 @@ gitconfig_remotes_request(struct imsgbuf *ibuf, struct i = 0; TAILQ_FOREACH(node, §ions->fields, link) { - char *name, *end; + char *name, *end, *mirror; if (strncasecmp("remote \"", node->field, 8) != 0) continue; @@ -129,6 +129,16 @@ gitconfig_remotes_request(struct imsgbuf *ibuf, struct err = got_error(GOT_ERR_GITCONFIG_SYNTAX); goto done; } + + remotes[i].mirror_references = 0; + mirror = got_gitconfig_get_str(gitconfig, node->field, + "mirror"); + if (mirror != NULL && + (strcasecmp(mirror, "true") == 0 || + strcasecmp(mirror, "on") == 0 || + strcasecmp(mirror, "yes") == 0 || + strcmp(mirror, "1") == 0)) + remotes[i].mirror_references = 1; i++; }