Coverage for virtuoso_utilities / native_entrypoint.py: 51%

1#!/usr/bin/env python3 

2 

3# SPDX-FileCopyrightText: 2025 Arcangelo Massari <arcangelo.massari@unibo.it> 

4# 

5# SPDX-License-Identifier: ISC 

6 

7""" 

8Virtuoso native mode entrypoint. 

9 

10Wrapper script that configures virtuoso.ini from environment variables, 

11then execs the original Virtuoso entrypoint. Designed for use as a Docker 

12container entrypoint. 

13 

14Usage in Dockerfile: 

15 FROM openlink/virtuoso-opensource-7:latest 

16 RUN pip install virtuoso-utilities 

17 ENTRYPOINT ["virtuoso-native-launch"] 

18 

19Environment variables: 

20 VIRTUOSO_MEMORY: Memory limit (e.g., "8g"). Default: 2/3 of available RAM 

21 VIRTUOSO_DBA_PASSWORD: DBA password. Also accepts DBA_PASSWORD for compatibility 

22 VIRTUOSO_ESTIMATED_DB_SIZE_GB: Estimated DB size for MaxCheckpointRemap 

23 VIRTUOSO_PARALLEL_THREADS: CPU cores for query parallelization 

24 VIRTUOSO_ENABLE_WRITE_PERMISSIONS: Enable SPARQL write ("true"/"1") 

25 VIRTUOSO_NUMBER_OF_BUFFERS: Override automatic buffer calculation 

26 VIRTUOSO_MAX_DIRTY_BUFFERS: Override automatic dirty buffer calculation 

27 VIRTUOSO_DATA_DIR: Data directory path 

28 VIRTUOSO_EXTRA_DIRS_ALLOWED: Additional DirsAllowed paths (comma-separated) 

29 VIRTUOSO_ORIGINAL_ENTRYPOINT: Original entrypoint to exec 

30""" 

31 

32import os 

33import sys 

34 

35from virtuoso_utilities.launch_virtuoso import ( 

36 DEFAULT_CONTAINER_DATA_DIR, 

37 DEFAULT_DIRS_ALLOWED, 

38 bytes_to_docker_mem_str, 

39 calculate_max_query_mem, 

40 calculate_threading_config, 

41 get_default_memory, 

42 get_optimal_buffer_values, 

43 get_virt_env_vars, 

44 grant_write_permissions, 

45 update_ini_memory_settings, 

46 wait_for_virtuoso_ready, 

47) 

48 

49ENV_MEMORY = "VIRTUOSO_MEMORY" 

50ENV_DBA_PASSWORD = "VIRTUOSO_DBA_PASSWORD" 

51ENV_ESTIMATED_DB_SIZE_GB = "VIRTUOSO_ESTIMATED_DB_SIZE_GB" 

52ENV_PARALLEL_THREADS = "VIRTUOSO_PARALLEL_THREADS" 

53ENV_ENABLE_WRITE_PERMISSIONS = "VIRTUOSO_ENABLE_WRITE_PERMISSIONS" 

54ENV_NUMBER_OF_BUFFERS = "VIRTUOSO_NUMBER_OF_BUFFERS" 

55ENV_MAX_DIRTY_BUFFERS = "VIRTUOSO_MAX_DIRTY_BUFFERS" 

56ENV_DATA_DIR = "VIRTUOSO_DATA_DIR" 

57ENV_EXTRA_DIRS_ALLOWED = "VIRTUOSO_EXTRA_DIRS_ALLOWED" 

58ENV_ORIGINAL_ENTRYPOINT = "VIRTUOSO_ORIGINAL_ENTRYPOINT" 

59 

60DEFAULT_ORIGINAL_ENTRYPOINT = "/virtuoso-entrypoint.sh" 

61 

62CGROUP_V2_MEMORY_MAX = "/sys/fs/cgroup/memory.max" 

63CGROUP_V1_MEMORY_LIMIT = "/sys/fs/cgroup/memory/memory.limit_in_bytes" 

64 

65 

66def get_container_memory_limit(): 

67 for path in [CGROUP_V2_MEMORY_MAX, CGROUP_V1_MEMORY_LIMIT]: 

68 try: 

69 with open(path) as f: 

70 value = f.read().strip() 

71 if value != "max": 

72 return int(value) 

73 except (FileNotFoundError, ValueError, PermissionError): 

74 continue 

75 return None 

76 

77 

78def get_native_default_memory(): 

79 container_limit = get_container_memory_limit() 

80 if container_limit: 

81 default_mem = max(int(container_limit * (2 / 3)), 1 * 1024**3) 

82 return bytes_to_docker_mem_str(default_mem) 

83 return get_default_memory() 

84 

85 

86def get_config_from_env(): 

87 config = {} 

88 config["memory"] = os.environ.get(ENV_MEMORY) or get_native_default_memory() 

89 config["dba_password"] = ( 

90 os.environ.get(ENV_DBA_PASSWORD) or os.environ.get("DBA_PASSWORD", "dba") 

91 ) 

92 config["estimated_db_size_gb"] = float( 

93 os.environ.get(ENV_ESTIMATED_DB_SIZE_GB, "0") 

94 ) 

95 threads_str = os.environ.get(ENV_PARALLEL_THREADS) 

96 config["parallel_threads"] = int(threads_str) if threads_str else None 

97 config["enable_write_permissions"] = os.environ.get( 

98 ENV_ENABLE_WRITE_PERMISSIONS, "" 

99 ).lower() in ("1", "true", "yes") 

100 buffers_str = os.environ.get(ENV_NUMBER_OF_BUFFERS) 

101 config["number_of_buffers"] = int(buffers_str) if buffers_str else None 

102 dirty_str = os.environ.get(ENV_MAX_DIRTY_BUFFERS) 

103 config["max_dirty_buffers"] = int(dirty_str) if dirty_str else None 

104 config["data_dir"] = os.environ.get(ENV_DATA_DIR, DEFAULT_CONTAINER_DATA_DIR) 

105 config["extra_dirs_allowed"] = os.environ.get(ENV_EXTRA_DIRS_ALLOWED, "") 

106 config["original_entrypoint"] = os.environ.get( 

107 ENV_ORIGINAL_ENTRYPOINT, DEFAULT_ORIGINAL_ENTRYPOINT 

108 ) 

109 return config 

110 

111 

112def configure_virtuoso(config): 

113 data_dir = config["data_dir"] 

114 ini_path = os.path.join(data_dir, "virtuoso.ini") 

115 

116 if config["number_of_buffers"] is None or config["max_dirty_buffers"] is None: 

117 num_buffers, max_dirty = get_optimal_buffer_values(config["memory"]) 

118 if config["number_of_buffers"] is None: 

119 config["number_of_buffers"] = num_buffers 

120 if config["max_dirty_buffers"] is None: 

121 config["max_dirty_buffers"] = max_dirty 

122 

123 dirs = DEFAULT_DIRS_ALLOWED.copy() 

124 dirs.add(data_dir) 

125 if config["extra_dirs_allowed"]: 

126 extra = set( 

127 d.strip() for d in config["extra_dirs_allowed"].split(",") if d.strip() 

128 ) 

129 dirs.update(extra) 

130 

131 threading = calculate_threading_config(config["parallel_threads"]) 

132 max_query_mem_value = calculate_max_query_mem( 

133 config["memory"], config["number_of_buffers"] 

134 ) 

135 update_ini_memory_settings( 

136 ini_path=ini_path, 

137 data_dir_path=data_dir, 

138 number_of_buffers=config["number_of_buffers"], 

139 max_dirty_buffers=config["max_dirty_buffers"], 

140 dirs_allowed=",".join(dirs), 

141 async_queue_max_threads=threading["async_queue_max_threads"], 

142 threads_per_query=threading["threads_per_query"], 

143 max_client_connections=threading["max_client_connections"], 

144 adjust_vector_size=0, 

145 vector_size=1000, 

146 checkpoint_interval=1, 

147 max_query_mem=max_query_mem_value, 

148 http_server_threads=threading["max_client_connections"], 

149 thread_cleanup_interval=1, 

150 resources_cleanup_interval=1, 

151 ) 

152 

153 print( 

154 f"Info: Configured Virtuoso with NumberOfBuffers={config['number_of_buffers']}, " 

155 f"MaxDirtyBuffers={config['max_dirty_buffers']}" 

156 ) 

157 

158 

159def set_virt_env_vars(config): 

160 env_vars = get_virt_env_vars( 

161 memory=config["memory"], 

162 number_of_buffers=config["number_of_buffers"], 

163 max_dirty_buffers=config["max_dirty_buffers"], 

164 parallel_threads=config["parallel_threads"], 

165 estimated_db_size_gb=config["estimated_db_size_gb"], 

166 ) 

167 for key, value in env_vars.items(): 

168 os.environ[key] = value 

169 

170 

171def apply_write_permissions_async(dba_password): 

172 pid = os.fork() 

173 if pid == 0: 

174 try: 

175 if wait_for_virtuoso_ready(dba_password): 

176 grant_write_permissions(dba_password) 

177 except Exception as e: 

178 print(f"Error applying write permissions: {e}", file=sys.stderr) 

179 os._exit(0) 

180 

181 

182def main(): 

183 config = get_config_from_env() 

184 

185 print("=" * 70) 

186 print("Virtuoso Native Mode Configuration") 

187 print(f" Memory: {config['memory']}") 

188 print(f" Data Dir: {config['data_dir']}") 

189 print(f" Write Permissions: {config['enable_write_permissions']}") 

190 print(f" Original Entrypoint: {config['original_entrypoint']}") 

191 print("=" * 70) 

192 

193 configure_virtuoso(config) 

194 set_virt_env_vars(config) 

195 

196 if config["enable_write_permissions"]: 

197 apply_write_permissions_async(config["dba_password"]) 

198 

199 remaining_args = sys.argv[1:] if len(sys.argv) > 1 else [] 

200 print(f"Info: Executing original entrypoint: {config['original_entrypoint']}") 

201 sys.stdout.flush() 

202 sys.stderr.flush() 

203 os.execve(config["original_entrypoint"], [config["original_entrypoint"]] + remaining_args, os.environ) 

204 

205 return 0 

206 

207 

208if __name__ == "__main__": 

209 sys.exit(main())