Merge pull request #126 from natefoo/no-root

Rootless processmanagerless mode

Author: mvdbeek

gravity/cli.py

@@ -7,6 +7,7 @@
 
 from gravity import io
 from gravity import options
+from gravity.settings import ProcessManager
 
 
 CONTEXT_SETTINGS = {
@@ -66,14 +67,19 @@ def get_command(self, ctx, name):
 @options.config_file_option()
 @options.state_dir_option()
 @options.no_log_option()
+@options.single_user_option()
 @click.pass_context
-def galaxy(ctx, debug, config_file, state_dir, quiet):
+def galaxy(ctx, debug, config_file, state_dir, quiet, single_user):
     """Run Galaxy server in the foreground"""
     set_debug(debug)
     ctx.cm_kwargs = {
         "config_file": config_file,
         "state_dir": state_dir,
+        "process_manager": ProcessManager.multiprocessing.value,
     }
+    if single_user:
+        os.environ["GALAXY_CONFIG_SINGLE_USER"] = single_user
+        os.environ["GALAXY_CONFIG_ADMIN_USERS"] = single_user
     mod = __import__("gravity.commands.cmd_start", None, None, ["cli"])
     return ctx.invoke(mod.cli, foreground=True, quiet=quiet)
 

gravity/config_manager.py

@@ -14,8 +14,15 @@
 from yaml import safe_load
 
 import gravity.io
-from gravity.settings import Settings
-from gravity.state import ConfigFile, service_for_service_type
+from gravity.settings import (
+    ProcessManager,
+    Settings,
+)
+from gravity.state import (
+    ConfigFile,
+    service_for_service_type,
+    galaxy_installed,
+)
 from gravity.util import recursive_update
 
 log = logging.getLogger(__name__)
@@ -35,22 +42,28 @@
 
 
 @contextlib.contextmanager
-def config_manager(config_file=None, state_dir=None, user_mode=None):
-    yield ConfigManager(config_file=config_file, state_dir=state_dir, user_mode=user_mode)
+def config_manager(config_file=None, state_dir=None, user_mode=None, process_manager=None):
+    yield ConfigManager(
+        config_file=config_file,
+        state_dir=state_dir,
+        user_mode=user_mode,
+        process_manager=process_manager
+    )
 
 
 class ConfigManager(object):
     galaxy_server_config_section = "galaxy"
     gravity_config_section = "gravity"
     app_config_file_option = "galaxy_config_file"
 
-    def __init__(self, config_file=None, state_dir=None, user_mode=None):
+    def __init__(self, config_file=None, state_dir=None, user_mode=None, process_manager=None):
         self.__configs = {}
         self.state_dir = None
         if state_dir is not None:
             # convert from pathlib.Path
             self.state_dir = str(state_dir)
         self.user_mode = user_mode
+        self.process_manager = process_manager or ProcessManager.supervisor.value
 
         gravity.io.debug(f"Gravity state dir: {state_dir}")
 
@@ -151,12 +164,13 @@ def __load_config(self, gravity_config_dict, app_config):
                 f"{self.__configs[gravity_settings.instance_name].gravity_config_file}")
             gravity.io.exception(f"Duplicate instance name {gravity_settings.instance_name}, instance names must be unique")
 
-        gravity_config_file = gravity_config_dict["__file__"]
+        gravity_config_file = gravity_config_dict.get("__file__")
         galaxy_config_file = app_config.get("__file__", gravity_config_file)
         galaxy_root = gravity_settings.galaxy_root or app_config.get("root")
 
         # TODO: document that the default state_dir is data_dir/gravity and that setting state_dir overrides this
-        gravity_data_dir = self.state_dir or os.path.join(app_config.get("data_dir", "database"), "gravity")
+        default_data_dir = "data" if galaxy_installed else "database"
+        gravity_data_dir = self.state_dir or os.path.join(app_config.get("data_dir", default_data_dir), "gravity")
         log_dir = gravity_settings.log_dir or os.path.join(gravity_data_dir, "log")
 
         # TODO: this should use galaxy.util.properties.load_app_properties() so that env vars work
@@ -175,7 +189,7 @@ def __load_config(self, gravity_config_dict, app_config):
             gravity_config_file=gravity_config_file,
             galaxy_config_file=galaxy_config_file,
             instance_name=gravity_settings.instance_name,
-            process_manager=gravity_settings.process_manager,
+            process_manager=gravity_settings.process_manager or self.process_manager,
             service_command_style=gravity_settings.service_command_style,
             app_server=gravity_settings.app_server,
             virtualenv=gravity_settings.virtualenv,
@@ -212,7 +226,7 @@ def create_static_handler_services(self, config: ConfigFile, app_config: dict):
             job_config = app_config["job_config"]
         else:
             # config in an external file
-            config_dir = os.path.dirname(config.galaxy_config_file)
+            config_dir = os.path.dirname(config.galaxy_config_file or os.getcwd())
             job_config = app_config.get("job_config_file")
             if not job_config:
                 for job_config in [os.path.abspath(os.path.join(config_dir, c)) for c in DEFAULT_JOB_CONFIG_FILES]:
@@ -383,9 +397,15 @@ def auto_load(self):
                 *glob.glob("/etc/galaxy/gravity.d/*.yaml"),
             )
         else:
-            configs = (os.path.join("config", "galaxy.yml"), os.path.join("config", "galaxy.yml.sample"))
+            configs = (os.path.join("config", "galaxy.yml"), "galaxy.yml", os.path.join("config", "galaxy.yml.sample"))
+            configs = tuple(config for config in configs if os.path.exists(config))
+        if not configs and galaxy_installed:
+            gravity.io.warn(
+                "Warning: No configuration file found but Galaxy is installed in this Python environment, running with "
+                "default config. Use -c / --config-file or set $GALAXY_CONFIG_FILE to specify a config file."
+            )
+            self.__load_config({}, {})
         for config in configs:
-            if os.path.exists(config):
-                self.load_config_file(os.path.abspath(config))
-                if not load_all:
-                    return
+            self.load_config_file(os.path.abspath(config))
+            if not load_all:
+                return

gravity/io.py

@@ -34,7 +34,7 @@ def error(message, *args):
 def warn(message, *args):
     if args:
         message = message % args
-    click.echo(click.style(message, fg="red"), err=True)
+    click.echo(click.style(message, fg="yellow"), err=True)
 
 
 def exception(message):

gravity/options.py

@@ -31,6 +31,15 @@ def user_mode_option():
     )
 
 
+def single_user_option():
+    return click.option(
+        "-s",
+        "--single-user",
+        default=None,
+        help="Run Galaxy in single user mode with the specified account email"
+    )
+
+
 def no_log_option():
     return click.option(
         '--quiet', is_flag=True, default=False, help="Only output supervisor logs, do not include process logs"

gravity/process_manager/__init__.py

@@ -65,7 +65,7 @@ def decorator(self, *args, instance_names=None, **kwargs):
 
 
 class BaseProcessExecutionEnvironment(metaclass=ABCMeta):
-    def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None):
+    def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None, process_executor=None):
         self.config_manager = config_manager or ConfigManager(state_dir=state_dir, config_file=config_file, user_mode=user_mode)
         self.tail = which("tail")
 
@@ -88,7 +88,8 @@ def _service_format_vars(self, config, service, pm_format_vars=None):
             "server_name": service.service_name,
             "galaxy_umask": service.settings.get("umask") or config.umask,
             "galaxy_conf": config.galaxy_config_file,
-            "galaxy_root": config.galaxy_root,
+            # TODO: this is used as the runtime directory, but it should probably be something else
+            "galaxy_root": config.galaxy_root or os.getcwd(),
             "virtualenv_bin": virtualenv_bin,
             "gravity_data_dir": shlex.quote(config.gravity_data_dir),
             "app_config": config.app_config,
@@ -112,7 +113,9 @@ def _service_format_vars(self, config, service, pm_format_vars=None):
                 path = environment.get("PATH", self._service_default_path())
                 environment["PATH"] = ":".join([virtualenv_bin, path])
         else:
-            config_file = shlex.quote(config.gravity_config_file)
+            config_file_option = ""
+            if config.gravity_config_file:
+                config_file_option = f" --config-file {shlex.quote(config.gravity_config_file)}"
             # is there a click way to do this?
             galaxyctl = sys.argv[0]
             if galaxyctl.endswith(f"{os.path.sep}galaxy"):
@@ -124,7 +127,7 @@ def _service_format_vars(self, config, service, pm_format_vars=None):
             instance_number_opt = ""
             if service.count > 1:
                 instance_number_opt = f" --service-instance {pm_format_vars['instance_number']}"
-            format_vars["command"] = f"{galaxyctl} --config-file {config_file} exec{instance_number_opt} {config.instance_name} {service.service_name}"
+            format_vars["command"] = f"{galaxyctl}{config_file_option} exec{instance_number_opt} {config.instance_name} {service.service_name}"
             environment = {}
         format_vars["environment"] = self._service_environment_formatter(environment, format_vars)
 
@@ -273,7 +276,7 @@ def exec(self, config, service, service_instance_number=None, no_exec=False):
 
         cmd = shlex.split(format_vars["command"])
         env = {**dict(os.environ), **format_vars["environment"]}
-        cwd = format_vars["galaxy_root"]
+        cwd = format_vars["galaxy_root"] or os.getcwd()
 
         # ensure the data dir exists
         try:
@@ -291,10 +294,13 @@ def exec(self, config, service, service_instance_number=None, no_exec=False):
 
 
 class ProcessManagerRouter:
-    def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None, **kwargs):
-        self.config_manager = config_manager or ConfigManager(state_dir=state_dir, config_file=config_file, user_mode=user_mode)
-        self._load_pm_modules(**kwargs)
+    def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None, process_manager=None, **kwargs):
+        self.config_manager = config_manager or ConfigManager(state_dir=state_dir,
+                                                              config_file=config_file,
+                                                              user_mode=user_mode,
+                                                              process_manager=process_manager)
         self._process_executor = ProcessExecutor(config_manager=self.config_manager)
+        self._load_pm_modules(**kwargs)
 
     def _load_pm_modules(self, *args, **kwargs):
         self.process_managers = {}
@@ -304,7 +310,7 @@ def _load_pm_modules(self, *args, **kwargs):
                 for name in dir(mod):
                     obj = getattr(mod, name)
                     if not name.startswith("_") and inspect.isclass(obj) and issubclass(obj, BaseProcessManager) and obj != BaseProcessManager:
-                        pm = obj(*args, config_manager=self.config_manager, **kwargs)
+                        pm = obj(*args, config_manager=self.config_manager, process_executor=self._process_executor, **kwargs)
                         self.process_managers[pm.name] = pm
 
     def _instance_service_names(self, names):

gravity/process_manager/multiprocessing.py

@@ -0,0 +1,62 @@
+"""
+"""
+import multiprocessing
+
+from gravity.process_manager import BaseProcessManager, ProcessExecutor
+from gravity.settings import ProcessManager
+
+
+class MultiprocessingProcessManager(BaseProcessManager):
+
+    name = ProcessManager.multiprocessing
+
+    def __init__(self, process_executor=None, **kwargs):
+        super().__init__(**kwargs)
+
+        assert process_executor is not None, f"Process executor is required for {self.__class__.__name__}"
+        self.process_executor = process_executor
+        self.processes = []
+
+    def follow(self, configs=None, service_names=None, quiet=False):
+        """ """
+
+    def start(self, configs=None, service_names=None):
+        for config in configs:
+            for service in config.services:
+                process = multiprocessing.Process(target=self.process_executor.exec, args=(config, service))
+                process.start()
+                self.processes.append(process)
+        for process in self.processes:
+            process.join()
+
+    def pm(self, *args, **kwargs):
+        """ """
+
+    def stop(self, configs=None, service_names=None):
+        """ """
+
+    def _present_pm_files_for_config(self, config):
+        """ """
+
+    def _disable_and_remove_pm_files(self, pm_files):
+        """ """
+
+    def restart(self, configs=None, service_names=None):
+        """ """
+
+    def graceful(self, configs=None, service_names=None):
+        """ """
+
+    def status(self, configs=None, service_names=None):
+        """ """
+
+    def terminate(self):
+        """ """
+
+    def shutdown(self):
+        """ """
+
+    def update(self, configs=None, force=False, clean=False):
+        """ """
+
+    _service_environment_formatter = ProcessExecutor._service_environment_formatter

gravity/settings.py

@@ -34,6 +34,7 @@ class LogLevel(str, Enum):
 class ProcessManager(str, Enum):
     supervisor = "supervisor"
     systemd = "systemd"
+    multiprocessing = "multiprocessing"
 
 
 class ServiceCommandStyle(str, Enum):
@@ -296,12 +297,13 @@ class Settings(BaseSettings):
     ``uwsgi:`` section will be ignored if Galaxy is started via Gravity commands (e.g ``./run.sh``, ``galaxy`` or ``galaxyctl``).
     """
 
-    process_manager: ProcessManager = Field(
+    process_manager: Optional[ProcessManager] = Field(
         None,
         description="""
 Process manager to use.
 ``supervisor`` is the default process manager when Gravity is invoked as a non-root user.
 ``systemd`` is the default when Gravity is invoked as root.
+``multiprocessing`` is the default when Gravity is invoked as the foreground shortcut ``galaxy`` instead of ``galaxyctl``
 """)
 
     service_command_style: ServiceCommandStyle = Field(
@@ -421,8 +423,6 @@ def _process_manager_systemd_if_root(cls, v, values):
         if v is None:
             if os.geteuid() == 0:
                 v = ProcessManager.systemd.value
-            else:
-                v = ProcessManager.supervisor.value
         return v
 
     # disable service instances unless command style is gravity

gravity/state.py

@@ -10,6 +10,12 @@
 import time
 from typing import Any, Dict, List, Optional
 
+try:
+    import galaxy.config
+    import galaxy.version
+    galaxy_installed = True
+except ImportError:
+    galaxy_installed = False
 try:
     from pydantic.v1 import BaseModel, validator
 except ImportError:
@@ -28,7 +34,8 @@
 
 def relative_to_galaxy_root(cls, v, values):
     if not os.path.isabs(v):
-        v = os.path.abspath(os.path.join(values["galaxy_root"], v))
+        galaxy_root = values.get("galaxy_root") or os.getcwd()
+        v = os.path.abspath(os.path.join(galaxy_root, v))
     return v
 
 
@@ -41,8 +48,8 @@ class GracefulMethod(str, enum.Enum):
 
 class ConfigFile(BaseModel):
     app_config: Dict[str, Any]
-    gravity_config_file: str
-    galaxy_config_file: str
+    gravity_config_file: Optional[str]
+    galaxy_config_file: Optional[str]
     instance_name: str
     process_manager: ProcessManager
     service_command_style: ServiceCommandStyle
@@ -66,18 +73,23 @@ def path_hash(self):
 
     @property
     def galaxy_version(self):
-        galaxy_version_file = os.path.join(self.galaxy_root, "lib", "galaxy", "version.py")
-        with open(galaxy_version_file) as fh:
-            locs = {}
-            exec(fh.read(), {}, locs)
-            return locs["VERSION"]
+        if galaxy_installed:
+            return galaxy.version.VERSION
+        else:
+            galaxy_version_file = os.path.join(self.galaxy_root, "lib", "galaxy", "version.py")
+            with open(galaxy_version_file) as fh:
+                locs = {}
+                exec(fh.read(), {}, locs)
+                return locs["VERSION"]
 
     @validator("galaxy_root")
     def _galaxy_root_required(cls, v, values):
         if v is None:
-            galaxy_config_file = values["galaxy_config_file"]
+            galaxy_config_file = values.get("galaxy_config_file")
             if os.environ.get("GALAXY_ROOT_DIR"):
                 v = os.path.abspath(os.environ["GALAXY_ROOT_DIR"])
+            elif galaxy_installed:
+                v = None
             elif os.path.exists(os.path.join(os.path.dirname(galaxy_config_file), os.pardir, "lib", "galaxy")):
                 v = os.path.abspath(os.path.join(os.path.dirname(galaxy_config_file), os.pardir))
             elif galaxy_config_file.endswith(os.path.join("galaxy", "config", "sample", "galaxy.yml.sample")):