## # rpm macros for old systems < el5 # (defined if bcond_with does not already exist) # %if %{?bcond_with:0}%{!?bcond_with:1} %define nil %{!?nil} %define defined() %{expand:%%{?%{1}:1}%%{!?%{1}:0}} %define undefined() %{expand:%%{?%{1}:0}%%{!?%{1}:1}} %define with() %{expand:%%{?with_%{1}:1}%%{!?with_%{1}:0}} %define without() %{expand:%%{?with_%{1}:0}%%{!?with_%{1}:1}} %define bcond_with() %{expand:%%{?_with_%{1}:%%global with_%{1} 1}} %define bcond_without() %{expand:%%{!?_without_%{1}:%%global with_%{1} 1}} %endif ##################################### # define 'with' conditions %bcond_without lustre %bcond_without lhsm # mysql is not actually optional. configure whines: #checking for mysql_config... no #configure: error: MySQL must be installed %bcond_without mysql %bcond_with sqlite %bcond_with backup %bcond_with shook %bcond_with recovtools # macro for defining dependencies %define pkg_dependencies \ %if 1 || %{with lhsm} \ BuildRequires: libattr-devel \ %endif \ %if 1 || %{with lustre} \ BuildRequires: libattr-devel \ BuildRequires: lustre-client \ %if %{defined lpackage} \ Requires: %{lpackage} >= %{lversion} \ BuildRequires: %{lpackage} >= %{lversion} \ %endif \ %endif \ %if %{with mysql} \ BuildRequires: /usr/include/mysql/mysql.h \ %endif \ BuildRequires: mailx \ BuildRequires: autoconf \ BuildRequires: automake \ BuildRequires: libtool \ %{nil} # target install dir for web gui %define installdir_www /var/www/html ###### end of macro definitions ##### Name: robinhood Version: 2.5.5 Vendor: CEA, HPC department Prefix: %{_prefix} %if %{with lustre} %if %{defined lversion} %define config_dependant .lustre%{lversion} %else %define config_dependant .lustre %endif %endif %if %{with sqlite} %define db_dependant .sqlite %endif Release: 2%{?config_dependant}%{?db_dependant}%{?dist} Summary: Robinhood - Policy engine and accounting tool for large filesystems License: CeCILL-C Group: Applications/System Url: http://robinhood.sourceforge.net Source0: https://github.com/cea-hpc/robinhood/archive/%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) %description Robinhood is a tool for monitoring and purging file systems. It is designed to process all its tasks in parallel, so it is particulary adapted for managing large file systems with millions of entries and petabytes of data. Generated using options: '--with-purpose=LUSTRE_HSM' '--with-lustre=/scratch/git/chroma/chroma-dependencies/robinhood/lustre' %package adm Summary: admin/config helper for Robinhood PolicyEngine Group: Applications/System Release: 2.noarch %description adm This RPM provides admin/config helper for Robinhood PolicyEngine (command rbh-config) which is common to all robinhood flavors. %package webgui Summary: Web interface to vizualize filesystems stats Group: Applications/System Release: 2.noarch Requires: php, php-mysql, php-xml, php-gd, php-pdo %description webgui Web interface to vizualize filesystems stats. This uses robinhood database to display misc. user and group stats. %if %{with recovtools} %package recov-tools Summary: Tools for MDS recovery. Group: Applications/System BuildRequires: libattr-devel %description recov-tools Tools for MDS recovery. %endif #summary, description and dependencies of each purpose %if %{with lhsm} %define configure_purpose LUSTRE_HSM %define purpose lhsm %define purpose_svc robinhood-lhsm %define purpose_bin rbh-lhsm %package %{purpose} Summary: PolicyEngine for Lustre-HSM binding Group: Applications/System %pkg_dependencies Requires: %{name}-adm >= %{version} %description %{purpose} Monitor Lustre usage and trigger file migration and purges. Generated using options: @ac_configure_args@ %else # backup mode %if %{with backup} %define configure_purpose BACKUP %define purpose backup %define purpose_svc robinhood-backup %define purpose_bin rbh-backup %package %{purpose} Summary: PolicyEngine for filesystem backup to external storage Group: Applications/System %pkg_dependencies Requires: %{name}-adm >= %{version} %description %{purpose} Backup modified files to an external backend. Generated using options: @ac_configure_args@ %else # light HSM binding mode (using shook library) %if %{with shook} %define configure_purpose SHOOK %define purpose shook %define purpose_svc robinhood-shook %define purpose_bin rbh-shook %package %{purpose} Summary: PolicyEngine for simple HSM binding Group: Applications/System %pkg_dependencies Requires: %{name}-adm >= %{version} BuildRequires: shook-server >= 1.3.5 Requires: shook-server >= 1.3.5 %description %{purpose} Backup modified files to an external backend and automatically trigger data restore using shook library. Generated using options: @ac_configure_args@ %package %{purpose}-annex Summary: Annex tools for simple HSM binding Group: Applications/System %pkg_dependencies BuildRequires: shook-server >= 1.3.5 Requires: shook-server >= 1.3.5 %description %{purpose}-annex Annex tools for simple HSM binding. Can run on different nodes than robinhood daemon. Generated using options: @ac_configure_args@ %else # TMP FS MGR %define configure_purpose TMP_FS_MGR %define purpose tmpfs %define purpose_svc robinhood %define purpose_bin robinhood %package %{purpose} Summary: Accounting, reporting and purge tool for large filesystems Group: Applications/System %pkg_dependencies Requires: %{name}-adm >= %{version} %description %{purpose} Provides disk space fair-share, accounting, reporting facilities for any POSIX filesystem, with extra features for Lustre (OST usage management). Possibly trigger purge by LRU when free disk space is low. Generated using options: @ac_configure_args@ %endif %endif %endif %prep %setup -q -n robinhood-%{version} %build rm -fr lustre mkdir -p lustre/lustre/include ln -s /usr/include/lustre lustre/lustre/include/lustre ln -s /usr/include/linux lustre/lustre/include/linux ln -s /usr/include/libcfs lustre/lustre/include/libcfs ln -s /usr/lib64 lustre/lustre/utils ./autogen.sh ./configure --with-purpose=LUSTRE_HSM --with-lustre=$(pwd)/lustre %{?configure_flags:configure_flags} \ --mandir=%{_mandir} \ --libdir=%{_libdir} make -j 8 || make %install rm -rf $RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/robinhood.d/%{purpose}/templates mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/robinhood.d/%{purpose}/includes mkdir -p $RPM_BUILD_ROOT/%{_initrddir} $RPM_BUILD_ROOT/%{_prefix}/sbin/%{purpose_bin} -T $RPM_BUILD_ROOT/%{_sysconfdir}/robinhood.d/%{purpose}/templates/%{purpose}_detailed.conf %if %{defined suse_version} install -m 755 scripts/robinhood.init.sles $RPM_BUILD_ROOT/%{_initrddir}/%{purpose_svc} %else install -m 755 scripts/robinhood.init $RPM_BUILD_ROOT/%{_initrddir}/%{purpose_svc} %endif mkdir $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig install -m 644 scripts/sysconfig_robinhood $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/robinhood-lhsm mkdir -p $RPM_BUILD_ROOT/%{installdir_www}/robinhood cp -r web_gui/acct/* $RPM_BUILD_ROOT/%{installdir_www}/robinhood/. cp web_gui/acct/.htaccess $RPM_BUILD_ROOT/%{installdir_www}/robinhood/. %clean rm -rf $RPM_BUILD_ROOT %post %{purpose} if [ -x %{_initrddir}/%{purpose_svc} ]; then if %{_initrddir}/%{purpose_svc} status | grep running | grep -v "not running" >/dev/null 2>&1; then %{_initrddir}/%{purpose_svc} stop WASRUNNING=1 fi [ -x /sbin/chkconfig ] && /sbin/chkconfig --del %{purpose_svc} [ -x /sbin/chkconfig ] && /sbin/chkconfig --add %{purpose_svc} if test x$WASRUNNING = x1; then %{_initrddir}/%{purpose_svc} start fi fi %preun if [ "$1" = 0 ]; then if [ -x %{_initrddir}/%{purpose_svc} ]; then [ -x /sbin/chkconfig ] && /sbin/chkconfig --del %{purpose_svc} if %{_initrddir}/%{purpose_svc} status | grep running | grep -v "not running" >/dev/null 2>&1; then %{_initrddir}/%{purpose_svc} stop fi fi fi %files adm %{_sbindir}/rbh-config %if %{with recovtools} %files recov-tools %{_sbindir}/*lovea %{_sbindir}/gen_lov_objid %{_sbindir}/ost_fids_remap %endif %if %{with shook} %files %{purpose}-annex %{_sbindir}/%{purpose_bin}-rebind %endif %files webgui # set apache permissions %defattr(750,root,apache) %{installdir_www}/robinhood %files %{purpose} %defattr(-,root,root,-) #%doc README #%doc COPYING #%doc ChangeLog # everythink in sbin, except rbh-config which is in adm %if %{with shook} %exclude %{_sbindir}/%{purpose_bin}-rebind %{_sbindir}/%{purpose_bin}-import %{_sbindir}/%{purpose_bin}-recov %{_sbindir}/%{purpose_bin}-undo-rm %endif %if %{with backup} %{_sbindir}/%{purpose_bin}-import %{_sbindir}/%{purpose_bin}-recov %{_sbindir}/%{purpose_bin}-rebind %{_sbindir}/%{purpose_bin}-undo-rm %endif %{_sbindir}/%{purpose_bin} %{_sbindir}/*-report %{_sbindir}/*-diff %{_bindir}/*-du %{_bindir}/*-find %if %{with backup} %{_sbindir}/rbhext_* %endif %if %{with shook} %{_sbindir}/rbhext_* %endif %{_mandir}/man1/* %config(noreplace) %{_sysconfdir}/sysconfig/%{purpose_svc} %dir %{_sysconfdir}/robinhood.d %dir %{_sysconfdir}/robinhood.d/%{purpose} %dir %{_sysconfdir}/robinhood.d/%{purpose}/includes %dir %{_sysconfdir}/robinhood.d/%{purpose}/templates %config %{_sysconfdir}/robinhood.d/%{purpose}/templates/%{purpose}_detailed.conf %{_initrddir}/%{purpose_svc} %changelog * Wed Jun 03 2015 Thomas Leibovici 2.5.5 - [lustre] Support Lustre versions up to 2.7. - [DB performance] Allow both batching and parallelizing DB operations when accounting is OFF. Benchmarks show a x3~x4 speedup of DB ingest rate in this case. - [DB performance] Reduce lock contention on indexes of STRIPE_ITEMS table. - [acct] Avoid accounting stats inconsistency when this feature is enabled/disabled/re-enabled. - [policy performance] Disable ignored class re-matching by default. - [policy performance] Disable sort by mtime by default. - [policy performance] Allow disabling sort by atime for purge policies. - [fix] Fix segfault when log rotation occurs under heavy logging. - [fix] When processing a RENAME changelog record, match the new path in filesystem instead of building it from DB (which caused 'incomplete path' errors). - [fix] Fix sendfile copy operation for backup mode. - [fix] Fix various requests about directories: --topdirs with --filter-path option, --toprmdir, ... - [fix] Fix possible duplicate insert error when updated entry attributes are only in ANNEX_INFO table (rare). - [fix] Some purge parameters may not be taken into account. - [packaging] clean permissions in distribution tarball. - [report performance] Improve performance of path matching when running directory reports with --filter-path option. * Mon Dec 08 2014 Thomas Leibovici 2.5.4 - [lustre] Lustre 2.4+: detect all stripe changes and update the DB accordingly. - [scan] Prevent from dropping entries from DB when opendir or stat fail. - [DB] Optimization to batch more DB requests. - [DB] Allow using any MySQL engine (new config parameter: listmanager::mysql::engine). - [rbh-config] Avoid backup_db to lock the whole tables with innodb. - [bugfix] Improve robustness to corrupted mtimes. - [bugfix] Fix possible crash in db_exec_sql. - [bugfix] Fix possible overflow when executing a custom archive command. - [backup mode] Clean all non-printable characters in backend path. - [pkg] libattr-devel is now mandatory to build robinhood. * Tue Jul 29 2014 Thomas Leibovici 2.5.3 - [bugfix] custom purge_command: fixed vulnerability to malicious file names. - [bugfix] changelog processing: fixed errors 'Entry has incomplete path in DB' in some case of rm/create patterns. - [bugfix] migration policy (fix): don't trigger copy of files that no longer exist. - [feature] rbh-config: new option 'reset_acct' to rebuild accounting info. - [feature] changelog reader: new parameter 'dump_file' to dump all incoming changelog records to a file. - [compat] port to Lustre 2.6.0. * Wed May 21 2014 Thomas Leibovici 2.5.2 - rbh-du: fixed major performance regression (since v2.5.0). - rbh-find: fixed occasional crash. - HSM and backup modes: fixed a risk of removing an existing entry from the backend (in some situations of hardlink/rename+unlink). - backup mode: optimized sendfile()-based copy (Linux kernel >= 2.6.33). - logs: avoid flood of log messages in case of DB connection error. - alerts: added host name to alert mail title. - rbh-config empty_db/repair_db: also manage/fix stored procedures. - cosmetic: fix wrong display of purged blocks for count-based triggers. - cosmetic: fix migration counter display. - init script: check that 'ulimit -s' is reasonable. - fixed build dependancies on Fedora19 and Fedora20. - code sanity: fixed many 'coverity' warnings + a couple of minor memleaks. - doc: details about RPM installation locations. - doc: detail of 'backend' paramaters for backup mode. * Wed Mar 19 2014 Thomas Leibovici 2.5.1 - entry processing (major fix): fixed deadlock when the pipeline is full and an entry with an unknown parent is encountered. - purge (enhancement): start purging data from the most used OSTs. - rbh-find (features): new options: -pool, -exec, -print, -nouser, -nogroup, -lsost - rbh-find (optimization): automatically switch to bulk DB request mode when command argument is filesystem root (+new option -nobulk to disable it). - logging (enhancement): new config parameters to control log header format - backup (feature): allow compressing data in archive. - backup (fix): wrong path in archive when robinhood root directory != mount point. - backup (fix): fix segfault when importing a single file with a FID-ending name. * Thu Feb 13 2014 Thomas Leibovici 2.5.0-1 Summary: - filesystem disaster recovery features - new namespace management (new DB schema to properly handle hardlinks, renames...) - scanning and changelog processing optimizations - database optimizations (requests batching) - many other changes, improvements and code cleaning... Details: - rbh-diff: * new command to detect differences between the filesystem and the information in robinhood database. * option "--apply=fs" for disaster recovery purpose: restore the filesystem metadata from robinhood DB. * makes it possible to rebuild a Lustre MDT from scratch, or from a LVM snapshot (see "Robinhood Lustre disaster recovery guide" for more details). - database: * new namespace implementation in database with new NAMES table (Cray contribution) - fixes/improves hardlink support - fixes/improves Lustre ChangeLog hardlink/rename/unlink support - saves DB storage space * database request batching: significantly increase database ingest rate. No longer needs innodb_flush_log_at_tx_commit != 1 to speed up DB operations. * additional information in DB that can help for disaster recovery: symlink info, access rights, stripe object indexes, stripe order, nlink... * set default commit behavior to transaction (prevent from DB inconsistencies) * optimized multi-table requests * optimization: minimized attribute set in DB update operations (don't update attributes that didn't change) * Fix: deal with mysql case insensitivity for string matching * triggers and stored procedures versioning mechanism * prevent from overflows for large INSERT requests, wide stripes... * prevent from DB deadlocks - scanning: * --partial-scan option is deprecated and replaced by an optional argument to --scan (e.g. --scan=/fs/subdir). * better management of partial scans: - better detection of removed entries vs. entries moved from a directory to another. - partial scans can be used for initial DB population (even if the DB is initially empty). * dealing with dead/deactivated OSTs: don't remove entries from DB if stat() returns ESHUTDOWN. * garbage collection of removed entries in DB is a long operation when terminating a scan (and even more when terminating a partial scan). Added --no-gc option to skip it (recommanded for partial scans). * automatically enabling --no-gc if the DB is initially empty (eg. for initial scan). * optimization: use *at() functions (openat, fstatat) and readdir by chunk (using getdents) instead of POSIX lstat() and readdir_r(). * optimization: use NOATIME flag to access entries as much as possible * optimizations of get_stripe and get_fid operations. * new --diff option for robinhood --scan and --readlog: output detected changes in a diff-like format. - Lustre changelogs: * changelog batching (Cray contribution): to speed up changelog processing, robinhood retains changelog records in memory a short time, to aggregate similar/redundant Changelog records on the same entry before updating its database. * support multiple changelog readers (for DNE) as multiple threads (default) or as multiple processes, possibly on different hosts, by giving a MDT index to --readlog option. * resilience to filesystem umount/mount. - rbh-report: * new option --entry-info to get all the stored information about an entry * option --dump-ost can now list multiple OSTs and support ranges notation (e.g. 3,5-8,12-23). * --dump-ost output indicates if a file has data on a given OST (could be striped on the OST but have no data on it). - rbh-find: * new option -crtime to filter entries on creation time. * output ordering closer to find output * added missing info in 'rbh-find -ls' output (nlink, mode, symlink info...) - robinhood-backup: * by default, use a built-in copy function to avoid the cost of forking copy commands. * rbh-backup-rebind: tool to rebind an entry in the backend if its fid changed in the filesystem for any reason (file copied to a new one to change its stripe, etc...) * rbh-backup-recov new features and options: --list (list information about entries to be recovered) --ost to only recover entries for a given set of OSTs (support range notation): the basic use-case is OST disaster recovery. --since