From 3eeea5b376c5e0eceb5378bbbc3876ddbfbdca7c Mon Sep 17 00:00:00 2001 From: Collin Funk Date: Fri, 15 Mar 2024 06:38:48 -0700 Subject: [PATCH] gnulib-tool.py: Follow gnulib-tool changes, part 58. Follow gnulib-tool change 2017-05-21 Bruno Haible gnulib-tool: Add options to create hard links. * pygnulib/GLConfig.py (GLConfig.__init__): Add 'hardlink' and 'lhardlink' to the parameter list. Initialize them. (GLConfig.default): Define the new options defaults to a boolean False. Don't use links by default. (GLConfig.checkHardlink, GLConfig.setHardlink, GLConfig.resetHardlink): New functions to manipulate and check whether the --hardlink option was given. (GLConfig.checkLHardlink, GLConfig.setLHardlink) (GLConfig.resetLHardlink): New functions to manipulate and check whether the --local-hardlink option was given. * pygnulib/GLFileSystem.py (CopyAction.Hardlink): New Enum value to describe hard links. (GLFileSystem.shouldLink): Check if hard links should be used. (GLFileAssistant.add, GLFileAssistant.update): Try to hard link if enabled. Copy the file if linking fails. (GLFileAssistant.add_or_update): Remove temporary files unconditionally. * pygnulib/GLInfo.py (GLInfo.usage): Document new options in the usage message. * pygnulib/GLTestDir.py (GLTestDir.execute): Try to hard link if enabled. Copy the file if linking fails. * pygnulib/constants.py (hardlink): New function based on symlink_relative. * pygnulib/main.py (main): Add new options --hardlink and --local-hardlink. Prefer the last given option when choosing symlinks or hard links. Invoke 'git update-index --refresh' to mitigate the effects of the hard links on git. --- ChangeLog | 32 ++++++++++++++++++++++ gnulib-tool.py.TODO | 31 +-------------------- pygnulib/GLConfig.py | 53 +++++++++++++++++++++++++++++++++--- pygnulib/GLFileSystem.py | 50 ++++++++++++++++++++++++---------- pygnulib/GLInfo.py | 4 +++ pygnulib/GLTestDir.py | 4 ++- pygnulib/constants.py | 20 ++++++++++++++ pygnulib/main.py | 59 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 204 insertions(+), 49 deletions(-) diff --git a/ChangeLog b/ChangeLog index 08a8276629..f0d8f33956 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,35 @@ +2024-03-15 Collin Funk + + gnulib-tool.py: Follow gnulib-tool changes, part 58. + Follow gnulib-tool change + 2017-05-21 Bruno Haible + gnulib-tool: Add options to create hard links. + * pygnulib/GLConfig.py (GLConfig.__init__): Add 'hardlink' and + 'lhardlink' to the parameter list. Initialize them. + (GLConfig.default): Define the new options defaults to a boolean False. + Don't use links by default. + (GLConfig.checkHardlink, GLConfig.setHardlink, GLConfig.resetHardlink): + New functions to manipulate and check whether the --hardlink option was given. + (GLConfig.checkLHardlink, GLConfig.setLHardlink) + (GLConfig.resetLHardlink): New functions to manipulate and check whether + the --local-hardlink option was given. + * pygnulib/GLFileSystem.py (CopyAction.Hardlink): New Enum value to + describe hard links. + (GLFileSystem.shouldLink): Check if hard links should be used. + (GLFileAssistant.add, GLFileAssistant.update): Try to hard link if + enabled. Copy the file if linking fails. + (GLFileAssistant.add_or_update): Remove temporary files unconditionally. + * pygnulib/GLInfo.py (GLInfo.usage): Document new options in the usage + message. + * pygnulib/GLTestDir.py (GLTestDir.execute): Try to hard link if + enabled. Copy the file if linking fails. + * pygnulib/constants.py (hardlink): New function based on + symlink_relative. + * pygnulib/main.py (main): Add new options --hardlink and + --local-hardlink. Prefer the last given option when choosing symlinks or + hard links. Invoke 'git update-index --refresh' to mitigate the effects + of the hard links on git. + 2024-03-14 Collin Funk gnulib-tool.py: Follow gnulib-tool changes, part 57. diff --git a/gnulib-tool.py.TODO b/gnulib-tool.py.TODO index b5796b24a3..eb315de832 100644 --- a/gnulib-tool.py.TODO +++ b/gnulib-tool.py.TODO @@ -16,10 +16,7 @@ The following commits to gnulib-tool have not yet been reflected in -------------------------------------------------------------------------------- Implement the options: - -h | --hardlink - --local-hardlink - -S | --more-symlinks - -H | --more-hardlinks + --automake-subdir-tests --help (same output) Optimize: @@ -198,32 +195,6 @@ Date: Thu Jun 8 15:09:31 2017 +0200 -------------------------------------------------------------------------------- -commit 306be564ba47ec412ca158f66ffa90a058f5253b -Author: Bruno Haible -Date: Mon May 22 01:39:59 2017 +0200 - - gnulib-tool: Add options to create hard links. - - * gnulib-tool (func_usage): Document options --hardlink, - --local-hardlink, --more-hardlinks. - (func_symlink): Renamed from func_ln. - (func_symlink_if_changed): Renamed from func_ln_if_changed. - (func_hardlink): New function. - (copymode, lcopymode): New variables. - (symbolic, lsymbolic): Remove variables. - (Options): Implement options --hardlink, --local-hardlink, - --more-hardlinks. - (func_should_link): Renamed from func_should_symlink. Set copyaction. - (func_add_file, func_update_file): Update invocation of - func_should_link. Invoke func_hardlink when appropriate. - (func_import): Update comments. - (func_create_testdir): Update invocation of func_should_link. Invoke - func_hardlink when appropriate. - Finally, invoke 'git update-index --refresh' to mitigate the effects of - the hard links on git. - --------------------------------------------------------------------------------- - commit f5142421c62024efa22cd4429100c4d9c1cc2ac4 Author: Bruno Haible Date: Sat May 20 13:24:37 2017 +0200 diff --git a/pygnulib/GLConfig.py b/pygnulib/GLConfig.py index fea65803d9..01b7c3b38f 100644 --- a/pygnulib/GLConfig.py +++ b/pygnulib/GLConfig.py @@ -59,9 +59,9 @@ class GLConfig(object): lgpl=None, gnu_make=None, makefile_name=None, tests_makefile_name=None, automake_subdir=None, libtool=None, conddeps=None, macro_prefix=None, podomain=None, witness_c_macro=None, vc_files=None, symbolic=None, - lsymbolic=None, configure_ac=None, ac_version=None, - libtests=None, single_configure=None, verbose=None, dryrun=None, - errors=None): + lsymbolic=None, hardlink=None, lhardlink=None, configure_ac=None, + ac_version=None, libtests=None, single_configure=None, verbose=None, + dryrun=None, errors=None): '''GLConfig.__init__(arguments) -> GLConfig Create new GLConfig instance.''' @@ -179,6 +179,14 @@ class GLConfig(object): self.resetLSymbolic() if lsymbolic != None: self.setLSymbolic(lsymbolic) + # hardlink + self.resetHardlink() + if hardlink != None: + self.setHardlink(hardlink) + # lhardlink + self.resetLHardlink() + if lhardlink != None: + self.resetLHardlink() # configure_ac self.resetAutoconfFile() if configure_ac != None: @@ -285,7 +293,7 @@ class GLConfig(object): elif key in ['modules', 'avoids', 'tests', 'incl_test_categories', 'excl_test_categories']: return list() elif key in ['libtool', 'lgpl', 'gnu_make', 'automake_subdir', 'conddeps', 'symbolic', - 'lsymbolic', 'libtests', 'dryrun']: + 'lsymbolic', 'hardlink', 'lhardlink', 'libtests', 'dryrun']: return False elif key == 'vc_files': return None @@ -1043,6 +1051,43 @@ class GLConfig(object): files from the local override directories.''' self.table['lsymbolic'] = False + # Define hardlink methods. + def checkHardlink(self) -> bool: + '''Check if pygnulib will make hard links instead of copying files.''' + return self.table['hardlink'] + + def setHardlink(self, value: bool) -> None: + '''Enable / disable creation of the hard links instead of copying files.''' + if type(value) is bool: + self.table['hardlink'] = value + else: # if type(value) is not bool + raise TypeError('value must be a bool, not %s' + % type(value).__name__) + + def resetHardlink(self) -> None: + '''Reset creation of the hard links instead of copying files.''' + self.table['hardlink'] = False + + # Define lhardlink methods. + def checkLHardlink(self) -> bool: + '''Check if pygnulib will make hard links instead of copying files, + only for files from the local override directories.''' + return self.table['lhardlink'] + + def setLHardlink(self, value: bool) -> None: + '''Enable / disable creation of hard links instead of copying files, + only for files from the local override directories.''' + if type(value) is bool: + self.table['lhardlink'] = value + else: # if type(value) is not bool + raise TypeError('value must be a bool, not %s' + % type(value).__name__) + + def resetLHardlink(self) -> None: + '''Reset creation of hard links instead of copying files, only for + files from the local override directories.''' + self.table['lhardlink'] = False + # Define verbosity methods. def getVerbosity(self): '''Get verbosity level.''' diff --git a/pygnulib/GLFileSystem.py b/pygnulib/GLFileSystem.py index f60e3fc6ea..90e3515096 100644 --- a/pygnulib/GLFileSystem.py +++ b/pygnulib/GLFileSystem.py @@ -43,6 +43,7 @@ copyfile = constants.copyfile movefile = constants.movefile isdir = os.path.isdir isfile = os.path.isfile +islink = os.path.islink #=============================================================================== @@ -51,6 +52,7 @@ isfile = os.path.isfile class CopyAction(Enum): Copy = 0 Symlink = 1 + Hardlink = 2 #=============================================================================== @@ -134,17 +136,25 @@ class GLFileSystem(object): def shouldLink(self, original, lookedup): '''GLFileSystem.shouldLink(original, lookedup) - Determines whether the original file should be copied or symlinked. + Determines whether the original file should be copied, symlinked, + or hardlinked. Returns a CopyAction.''' symbolic = self.config['symbolic'] lsymbolic = self.config['lsymbolic'] + hardlink = self.config['hardlink'] + lhardlink = self.config['lhardlink'] localpath = self.config['localpath'] if symbolic: return CopyAction.Symlink - if lsymbolic: + elif hardlink: + return CopyAction.Hardlink + if lsymbolic or lhardlink: for localdir in localpath: if lookedup == joinpath(localdir, original): - return CopyAction.Symlink + if lsymbolic: + return CopyAction.Symlink + else: # elif lhardlink + return CopyAction.Hardlink return CopyAction.Copy @@ -261,10 +271,14 @@ class GLFileAssistant(object): and not tmpflag and filecmp.cmp(lookedup, tmpfile): constants.link_if_changed(lookedup, joinpath(destdir, rewritten)) else: # if any of these conditions is not met - try: # Try to move file - movefile(tmpfile, joinpath(destdir, rewritten)) - except Exception as error: - raise GLError(17, original) + if self.filesystem.shouldLink(original, lookedup) == CopyAction.Hardlink \ + and not tmpflag and filecmp.cmp(lookedup, tmpfile): + constants.hardlink(lookedup, joinpath(destdir, rewritten)) + else: # Move instead of linking. + try: # Try to move file + movefile(tmpfile, joinpath(destdir, rewritten)) + except Exception as error: + raise GLError(17, original) else: # if self.config['dryrun'] print('Copy file %s' % rewritten) @@ -310,12 +324,16 @@ class GLFileAssistant(object): and not tmpflag and filecmp.cmp(lookedup, tmpfile): constants.link_if_changed(lookedup, basepath) else: # if any of these conditions is not met - try: # Try to move file - if os.path.exists(basepath): - os.remove(basepath) - copyfile(tmpfile, rewritten) - except Exception as error: - raise GLError(17, original) + if self.filesystem.shouldLink(original, lookedup) == CopyAction.Hardlink \ + and not tmpflag and filecmp.cmp(lookedup, tmpfile): + constants.hardlink(lookedup, basepath) + else: # Move instead of linking. + try: # Try to move file + if os.path.exists(basepath): + os.remove(basepath) + copyfile(tmpfile, joinpath(destdir, rewritten)) + except Exception as error: + raise GLError(17, original) else: # if self.config['dryrun'] if already_present: print('Update file %s (backup in %s)' % (rewritten, backupname)) @@ -372,11 +390,15 @@ class GLFileAssistant(object): file.write(data) path = joinpath(self.config['destdir'], rewritten) if isfile(path): + # The file already exists. self.update(lookedup, tmpflag, tmpfile, already_present) - os.remove(tmpfile) else: # if not isfile(path) + # Install the file. + # Don't protest if the file should be there but isn't: it happens + # frequently that developers don't put autogenerated files under version control. self.add(lookedup, tmpflag, tmpfile) self.addFile(rewritten) + os.remove(tmpfile) def super_update(self, basename, tmpfile): '''GLFileAssistant.super_update(basename, tmpfile) -> tuple diff --git a/pygnulib/GLInfo.py b/pygnulib/GLInfo.py index cb77c5e923..8851341c4d 100644 --- a/pygnulib/GLInfo.py +++ b/pygnulib/GLInfo.py @@ -315,10 +315,14 @@ Options for --import, --add/remove-import, --update, -s, --symbolic, --symlink Make symbolic links instead of copying files. --local-symlink Make symbolic links instead of copying files, only for files from the local override directory. + -h, --hardlink Make hard links instead of copying files. + --local-hardlink Make hard links instead of copying files, only + for files from the local override directory. Options for --import, --add/remove-import, --update: -S, --more-symlinks Deprecated; equivalent to --symlink. + -H, --more-hardlinks Deprecated; equivalent to --hardlink. Report bugs to .''' return result diff --git a/pygnulib/GLTestDir.py b/pygnulib/GLTestDir.py index f747a480f5..fd8978c667 100644 --- a/pygnulib/GLTestDir.py +++ b/pygnulib/GLTestDir.py @@ -344,7 +344,7 @@ class GLTestDir(object): for file in self.rewrite_files(filelist)] directories = sorted(set(directories)) - # Copy files or make symbolic links. + # Copy files or make symbolic links or hard links. filetable = list() for src in filelist: dest = self.rewrite_files([src])[-1] @@ -366,6 +366,8 @@ class GLTestDir(object): else: # if not flag if self.filesystem.shouldLink(src, lookedup) == CopyAction.Symlink: constants.link_relative(lookedup, destpath) + if self.filesystem.shouldLink(src, lookedup) == CopyAction.Hardlink: + constants.hardlink(lookedup, destpath) else: copyfile(lookedup, destpath) diff --git a/pygnulib/constants.py b/pygnulib/constants.py index a3bf8cd1d4..dc2e68069d 100644 --- a/pygnulib/constants.py +++ b/pygnulib/constants.py @@ -412,6 +412,26 @@ def link_if_changed(src, dest): symlink_relative(link_value, dest) +def hardlink(src: str, dest: str) -> None: + '''Like ln, except use cp -p if ln fails. + src is either absolute or relative to the directory of dest.''' + try: + os.link(src, dest) + except PermissionError: + sys.stderr.write('%s: ln failed; falling back on cp -p\n' % APP['name']) + if src.startswith('/') or (len(src) >= 2 and src[1] == ':'): + # src is absolute. + cp_src = src + else: + # src is relative to the directory of dest. + last_slash = dest.rfind('/') + if last_slash >= 0: + cp_src = joinpath(dest[0: last_slash - 1], src) + else: + cp_src = src + copyfile2(cp_src, dest) + + def filter_filelist(separator, filelist, prefix, suffix, removed_prefix, removed_suffix, added_prefix='', added_suffix=''): diff --git a/pygnulib/main.py b/pygnulib/main.py index 5d65a45b25..b4e777c0f4 100644 --- a/pygnulib/main.py +++ b/pygnulib/main.py @@ -445,6 +445,16 @@ def main(): dest='lsymlink', default=None, action='store_true') + # hardlink + parser.add_argument('-h', '--hardlink', + dest='hardlink', + default=None, + action='store_true') + # local-hardlink + parser.add_argument('--local-hardlink', + dest='lhardlink', + default=None, + action='store_true') # All other arguments are collected. parser.add_argument("non_option_arguments", nargs='*') @@ -750,9 +760,43 @@ def main(): for module in list1 ] symlink = cmdargs.symlink == True lsymlink = cmdargs.lsymlink == True + hardlink = cmdargs.hardlink == True + lhardlink = cmdargs.lhardlink == True single_configure = cmdargs.single_configure docbase = None + # --symlink and --hardlink are mutually exclusive. + # Use the last one given. + if symlink and hardlink: + optindex_table = {} + options = list(reversed(sys.argv[1:])) + options_count = len(options) + for option in ['-s', '--symbolic', '--symlink', '-h', '--hardlink']: + if option in options: + optindex_table[option] = options_count - options.index(option) + last_option = max(optindex_table, key=optindex_table.get) + # Disable the option that is not the last one. + if last_option in ['-s', '--symbolic', '--symlink']: + hardlink = False + else: # last_option is --hardlink or equivalent. + symlink = False + + # --local-symlink and --local-hardlink are mutually exclusive. + # Use the last one given. + if lsymlink and lhardlink: + optindex_table = {} + options = list(reversed(sys.argv[1:])) + options_count = len(options) + for option in ['--local-symlink', '--local-hardlink']: + if option in options: + optindex_table[option] = options_count - options.index(option) + last_option = max(optindex_table, key=optindex_table.get) + # Disable the option that is not the last one. + if last_option == '--local-symlink': + hardlink = False + else: # last_option is --local-hardlink. + symlink = False + # Create pygnulib configuration. config = classes.GLConfig( destdir=destdir, @@ -781,6 +825,8 @@ def main(): vc_files=vc_files, symbolic=symlink, lsymbolic=lsymlink, + hardlink=hardlink, + lhardlink=lhardlink, single_configure=single_configure, verbose=verbose, dryrun=dryrun, @@ -1269,6 +1315,19 @@ def main(): sys.stderr.write(message) sys.exit(1) + if hardlink or lhardlink: + # Setting hard links modifies the ctime of files in the gnulib checkout. + # This disturbs the result of the next "gitk" invocation. + # Workaround: Let git scan the files. This can be done through + # "git update-index --refresh" or "git status" or "git diff". + if isdir(joinpath(APP['root'], '.git')): + try: + sp.run(['git', 'update-index', '--refresh'], + cwd=APP['root'], stdout=sp.DEVNULL, stderr=sp.DEVNULL) + except Exception: + # We did our best... + pass + if __name__ == '__main__': try: # Try to execute -- 2.44.0