Coverage for virtuoso_utilities / native_entrypoint.py: 51%

1#!/usr/bin/env python3 

2""" 

3Virtuoso native mode entrypoint. 

4 

5Wrapper script that configures virtuoso.ini from environment variables, 

6then execs the original Virtuoso entrypoint. Designed for use as a Docker 

7container entrypoint. 

8 

9Usage in Dockerfile: 

10 FROM openlink/virtuoso-opensource-7:latest 

11 RUN pip install virtuoso-utilities 

12 ENTRYPOINT ["virtuoso-native-launch"] 

13 

14Environment variables: 

15 VIRTUOSO_MEMORY: Memory limit (e.g., "8g"). Default: 2/3 of available RAM 

16 VIRTUOSO_DBA_PASSWORD: DBA password. Also accepts DBA_PASSWORD for compatibility 

17 VIRTUOSO_ESTIMATED_DB_SIZE_GB: Estimated DB size for MaxCheckpointRemap 

18 VIRTUOSO_PARALLEL_THREADS: CPU cores for query parallelization 

19 VIRTUOSO_ENABLE_WRITE_PERMISSIONS: Enable SPARQL write ("true"/"1") 

20 VIRTUOSO_NUMBER_OF_BUFFERS: Override automatic buffer calculation 

21 VIRTUOSO_MAX_DIRTY_BUFFERS: Override automatic dirty buffer calculation 

22 VIRTUOSO_DATA_DIR: Data directory path 

23 VIRTUOSO_EXTRA_DIRS_ALLOWED: Additional DirsAllowed paths (comma-separated) 

24 VIRTUOSO_ORIGINAL_ENTRYPOINT: Original entrypoint to exec 

25""" 

26 

27import os 

28import sys 

29 

30from virtuoso_utilities.launch_virtuoso import ( 

31 DEFAULT_CONTAINER_DATA_DIR, 

32 DEFAULT_DIRS_ALLOWED, 

33 bytes_to_docker_mem_str, 

34 calculate_max_query_mem, 

35 calculate_threading_config, 

36 get_default_memory, 

37 get_optimal_buffer_values, 

38 get_virt_env_vars, 

39 grant_write_permissions, 

40 update_ini_memory_settings, 

41 wait_for_virtuoso_ready, 

42) 

43 

44ENV_MEMORY = "VIRTUOSO_MEMORY" 

45ENV_DBA_PASSWORD = "VIRTUOSO_DBA_PASSWORD" 

46ENV_ESTIMATED_DB_SIZE_GB = "VIRTUOSO_ESTIMATED_DB_SIZE_GB" 

47ENV_PARALLEL_THREADS = "VIRTUOSO_PARALLEL_THREADS" 

48ENV_ENABLE_WRITE_PERMISSIONS = "VIRTUOSO_ENABLE_WRITE_PERMISSIONS" 

49ENV_NUMBER_OF_BUFFERS = "VIRTUOSO_NUMBER_OF_BUFFERS" 

50ENV_MAX_DIRTY_BUFFERS = "VIRTUOSO_MAX_DIRTY_BUFFERS" 

51ENV_DATA_DIR = "VIRTUOSO_DATA_DIR" 

52ENV_EXTRA_DIRS_ALLOWED = "VIRTUOSO_EXTRA_DIRS_ALLOWED" 

53ENV_ORIGINAL_ENTRYPOINT = "VIRTUOSO_ORIGINAL_ENTRYPOINT" 

54 

55DEFAULT_ORIGINAL_ENTRYPOINT = "/virtuoso-entrypoint.sh" 

56 

57CGROUP_V2_MEMORY_MAX = "/sys/fs/cgroup/memory.max" 

58CGROUP_V1_MEMORY_LIMIT = "/sys/fs/cgroup/memory/memory.limit_in_bytes" 

59 

60 

61def get_container_memory_limit(): 

62 for path in [CGROUP_V2_MEMORY_MAX, CGROUP_V1_MEMORY_LIMIT]: 

63 try: 

64 with open(path) as f: 

65 value = f.read().strip() 

66 if value != "max": 

67 return int(value) 

68 except (FileNotFoundError, ValueError, PermissionError): 

69 continue 

70 return None 

71 

72 

73def get_native_default_memory(): 

74 container_limit = get_container_memory_limit() 

75 if container_limit: 

76 default_mem = max(int(container_limit * (2 / 3)), 1 * 1024**3) 

77 return bytes_to_docker_mem_str(default_mem) 

78 return get_default_memory() 

79 

80 

81def get_config_from_env(): 

82 config = {} 

83 config["memory"] = os.environ.get(ENV_MEMORY) or get_native_default_memory() 

84 config["dba_password"] = ( 

85 os.environ.get(ENV_DBA_PASSWORD) or os.environ.get("DBA_PASSWORD", "dba") 

86 ) 

87 config["estimated_db_size_gb"] = float( 

88 os.environ.get(ENV_ESTIMATED_DB_SIZE_GB, "0") 

89 ) 

90 threads_str = os.environ.get(ENV_PARALLEL_THREADS) 

91 config["parallel_threads"] = int(threads_str) if threads_str else None 

92 config["enable_write_permissions"] = os.environ.get( 

93 ENV_ENABLE_WRITE_PERMISSIONS, "" 

94 ).lower() in ("1", "true", "yes") 

95 buffers_str = os.environ.get(ENV_NUMBER_OF_BUFFERS) 

96 config["number_of_buffers"] = int(buffers_str) if buffers_str else None 

97 dirty_str = os.environ.get(ENV_MAX_DIRTY_BUFFERS) 

98 config["max_dirty_buffers"] = int(dirty_str) if dirty_str else None 

99 config["data_dir"] = os.environ.get(ENV_DATA_DIR, DEFAULT_CONTAINER_DATA_DIR) 

100 config["extra_dirs_allowed"] = os.environ.get(ENV_EXTRA_DIRS_ALLOWED, "") 

101 config["original_entrypoint"] = os.environ.get( 

102 ENV_ORIGINAL_ENTRYPOINT, DEFAULT_ORIGINAL_ENTRYPOINT 

103 ) 

104 return config 

105 

106 

107def configure_virtuoso(config): 

108 data_dir = config["data_dir"] 

109 ini_path = os.path.join(data_dir, "virtuoso.ini") 

110 

111 if config["number_of_buffers"] is None or config["max_dirty_buffers"] is None: 

112 num_buffers, max_dirty = get_optimal_buffer_values(config["memory"]) 

113 if config["number_of_buffers"] is None: 

114 config["number_of_buffers"] = num_buffers 

115 if config["max_dirty_buffers"] is None: 

116 config["max_dirty_buffers"] = max_dirty 

117 

118 dirs = DEFAULT_DIRS_ALLOWED.copy() 

119 dirs.add(data_dir) 

120 if config["extra_dirs_allowed"]: 

121 extra = set( 

122 d.strip() for d in config["extra_dirs_allowed"].split(",") if d.strip() 

123 ) 

124 dirs.update(extra) 

125 

126 threading = calculate_threading_config(config["parallel_threads"]) 

127 max_query_mem_value = calculate_max_query_mem( 

128 config["memory"], config["number_of_buffers"] 

129 ) 

130 update_ini_memory_settings( 

131 ini_path=ini_path, 

132 data_dir_path=data_dir, 

133 number_of_buffers=config["number_of_buffers"], 

134 max_dirty_buffers=config["max_dirty_buffers"], 

135 dirs_allowed=",".join(dirs), 

136 async_queue_max_threads=threading["async_queue_max_threads"], 

137 threads_per_query=threading["threads_per_query"], 

138 max_client_connections=threading["max_client_connections"], 

139 adjust_vector_size=0, 

140 vector_size=1000, 

141 checkpoint_interval=1, 

142 max_query_mem=max_query_mem_value, 

143 http_server_threads=threading["max_client_connections"], 

144 thread_cleanup_interval=1, 

145 resources_cleanup_interval=1, 

146 ) 

147 

148 print( 

149 f"Info: Configured Virtuoso with NumberOfBuffers={config['number_of_buffers']}, " 

150 f"MaxDirtyBuffers={config['max_dirty_buffers']}" 

151 ) 

152 

153 

154def set_virt_env_vars(config): 

155 env_vars = get_virt_env_vars( 

156 memory=config["memory"], 

157 number_of_buffers=config["number_of_buffers"], 

158 max_dirty_buffers=config["max_dirty_buffers"], 

159 parallel_threads=config["parallel_threads"], 

160 estimated_db_size_gb=config["estimated_db_size_gb"], 

161 ) 

162 for key, value in env_vars.items(): 

163 os.environ[key] = value 

164 

165 

166def apply_write_permissions_async(dba_password): 

167 pid = os.fork() 

168 if pid == 0: 

169 try: 

170 if wait_for_virtuoso_ready(dba_password): 

171 grant_write_permissions(dba_password) 

172 except Exception as e: 

173 print(f"Error applying write permissions: {e}", file=sys.stderr) 

174 os._exit(0) 

175 

176 

177def main(): 

178 config = get_config_from_env() 

179 

180 print("=" * 70) 

181 print("Virtuoso Native Mode Configuration") 

182 print(f" Memory: {config['memory']}") 

183 print(f" Data Dir: {config['data_dir']}") 

184 print(f" Write Permissions: {config['enable_write_permissions']}") 

185 print(f" Original Entrypoint: {config['original_entrypoint']}") 

186 print("=" * 70) 

187 

188 configure_virtuoso(config) 

189 set_virt_env_vars(config) 

190 

191 if config["enable_write_permissions"]: 

192 apply_write_permissions_async(config["dba_password"]) 

193 

194 remaining_args = sys.argv[1:] if len(sys.argv) > 1 else [] 

195 print(f"Info: Executing original entrypoint: {config['original_entrypoint']}") 

196 sys.stdout.flush() 

197 sys.stderr.flush() 

198 os.execve(config["original_entrypoint"], [config["original_entrypoint"]] + remaining_args, os.environ) 

199 

200 return 0 

201 

202 

203if __name__ == "__main__": 

204 sys.exit(main())