Coverage for pephubclient/helpers.py: 59%

173 statements  

« prev     ^ index     » next       coverage.py v7.15.3, created at 2026-08-04 18:15 +0000

1import json 

2from typing import Any, Callable, Optional, Union, Literal, Tuple 

3import peppy 

4import yaml 

5from pathlib import Path 

6import os 

7import pandas as pd 

8from peppy.const import ( 

9 NAME_KEY, 

10 DESC_KEY, 

11 CONFIG_KEY, 

12 SUBSAMPLE_RAW_LIST_KEY, 

13 SAMPLE_RAW_DICT_KEY, 

14 CFG_SAMPLE_TABLE_KEY, 

15 CFG_SUBSAMPLE_TABLE_KEY, 

16) 

17 

18import requests 

19from requests.exceptions import ConnectionError 

20from urllib.parse import urlencode 

21 

22from ubiquerg import parse_registry_path 

23from pydantic import ValidationError 

24 

25from pephubclient.exceptions import ( 

26 PEPExistsError, 

27 ResponseError, 

28 BasePephubclientException, 

29) 

30from pephubclient.constants import RegistryPath 

31from pephubclient.files_manager import FilesManager 

32from pephubclient.models import ProjectDict 

33 

34 

35class RequestManager: 

36 @staticmethod 

37 def send_request( 

38 method: str, 

39 url: str, 

40 headers: Optional[dict] = None, 

41 cookies: Optional[dict] = None, 

42 params: Optional[dict] = None, 

43 json: Optional[Union[dict, list]] = None, 

44 ) -> requests.Response: 

45 request_return = requests.request( 

46 method=method, 

47 url=url, 

48 verify=False, 

49 cookies=cookies, 

50 headers=headers, 

51 params=params, 

52 json=json, 

53 timeout=10, 

54 ) 

55 if request_return.status_code == 401: 

56 if ( 

57 RequestManager.decode_response(request_return, output_json=True).get( 

58 "detail" 

59 ) 

60 == "JWT has expired" 

61 ): 

62 raise ResponseError("JWT has expired. Please log in again.") 

63 return request_return 

64 

65 @staticmethod 

66 def decode_response( 

67 response: requests.Response, encoding: str = "utf-8", output_json: bool = False 

68 ) -> Union[str, dict]: 

69 """ 

70 Decode the response from GitHub and pack the returned data into appropriate model. 

71 

72 :param response: Response from GitHub. 

73 :param encoding: Response encoding [Default: utf-8] 

74 :param output_json: If True, return response in json format 

75 :return: Response data as an instance of correct model. 

76 """ 

77 

78 try: 

79 if output_json: 

80 return response.json() 

81 else: 

82 return response.content.decode(encoding) 

83 except json.JSONDecodeError as err: 

84 raise ResponseError(f"Error in response encoding format: {err}") 

85 

86 @staticmethod 

87 def parse_query_param(pep_variables: dict) -> str: 

88 """ 

89 Grab all the variables passed by user (if any) and parse them to match the format specified 

90 by PEPhub API for query parameters. 

91 

92 :param pep_variables: dict of query parameters 

93 :return: PEPHubClient variables transformed into string in correct format. 

94 """ 

95 return "?" + urlencode(pep_variables) 

96 

97 @staticmethod 

98 def parse_header(jwt_data: Optional[str] = None) -> dict: 

99 """ 

100 Create Authorization header 

101 

102 :param jwt_data: jwt string 

103 :return: Authorization dict 

104 """ 

105 if jwt_data: 

106 return {"Authorization": jwt_data} 

107 else: 

108 return {} 

109 

110 

111class MessageHandler: 

112 """ 

113 Class holding print function in different colors 

114 """ 

115 

116 RED = 9 

117 YELLOW = 11 

118 GREEN = 40 

119 

120 @staticmethod 

121 def print_error(text: str) -> None: 

122 print(f"\033[38;5;9m{text}\033[0m") 

123 

124 @staticmethod 

125 def print_success(text: str) -> None: 

126 print(f"\033[38;5;40m{text}\033[0m") 

127 

128 @staticmethod 

129 def print_warning(text: str) -> None: 

130 print(f"\033[38;5;11m{text}\033[0m") 

131 

132 

133def call_client_func(func: Callable[..., Any], **kwargs) -> Any: 

134 """ 

135 Catch exceptions in functions called through cli. 

136 

137 :param func: The function to call. 

138 :param kwargs: The keyword arguments to pass to the function. 

139 :return: The result of the function call. 

140 """ 

141 

142 try: 

143 func(**kwargs) 

144 except ConnectionError as err: 

145 MessageHandler.print_error(f"Failed to connect to server. Try later. {err}") 

146 except ResponseError as err: 

147 MessageHandler.print_error(f"{err}") 

148 except PEPExistsError as err: 

149 MessageHandler.print_warning(f"PEP already exists. {err}") 

150 except OSError as err: 

151 MessageHandler.print_error(f"{err}") 

152 

153 

154def is_registry_path(input_string: str) -> bool: 

155 """ 

156 Check if input is a registry path to pephub 

157 :param str input_string: path to the PEP (or registry path) 

158 :return bool: True if input is a registry path 

159 """ 

160 if input_string.endswith(".yaml"): 

161 return False 

162 try: 

163 RegistryPath(**parse_registry_path(input_string)) 

164 except (ValidationError, TypeError): 

165 return False 

166 return True 

167 

168 

169def unwrap_registry_path(input_string: str) -> RegistryPath: 

170 """ 

171 Unwrap registry path from string 

172 :param str input_string: path to the PEP (or registry path) 

173 :return RegistryPath: RegistryPath object 

174 """ 

175 return RegistryPath(**parse_registry_path(input_string)) 

176 

177 

178def _build_filename(registry_path: RegistryPath) -> str: 

179 """ 

180 Takes query string and creates output filename to save the project to. 

181 

182 :param registry_path: Query string that was used to find the project. 

183 :return: Filename uniquely identifying the project. 

184 """ 

185 filename = "_".join(filter(bool, [registry_path.namespace, registry_path.item])) 

186 if registry_path.tag: 

187 filename += f"_{registry_path.tag}" 

188 return filename 

189 

190 

191def _save_zip_pep(project: dict, zip_filepath: str, force: bool = False) -> None: 

192 """ 

193 Zip and save a project 

194 

195 :param project: peppy project to zip 

196 :param zip_filepath: path to save zip file 

197 :param force: overwrite project if exists 

198 """ 

199 

200 content_to_zip = {} 

201 config = project[CONFIG_KEY] 

202 project_name = config[NAME_KEY] 

203 

204 if project[SAMPLE_RAW_DICT_KEY] is not None: 

205 config[CFG_SAMPLE_TABLE_KEY] = ["sample_table.csv"] 

206 content_to_zip["sample_table.csv"] = pd.DataFrame( 

207 project[SAMPLE_RAW_DICT_KEY] 

208 ).to_csv(index=False) 

209 

210 if project[SUBSAMPLE_RAW_LIST_KEY] is not None: 

211 if not isinstance(project[SUBSAMPLE_RAW_LIST_KEY], list): 

212 config[CFG_SUBSAMPLE_TABLE_KEY] = ["subsample_table1.csv"] 

213 content_to_zip["subsample_table1.csv"] = pd.DataFrame( 

214 project[SUBSAMPLE_RAW_LIST_KEY] 

215 ).to_csv(index=False) 

216 else: 

217 config[CFG_SUBSAMPLE_TABLE_KEY] = [] 

218 for number, file in enumerate(project[SUBSAMPLE_RAW_LIST_KEY]): 

219 file_name = f"subsample_table{number + 1}.csv" 

220 config[CFG_SUBSAMPLE_TABLE_KEY].append(file_name) 

221 content_to_zip[file_name] = pd.DataFrame(file).to_csv(index=False) 

222 

223 content_to_zip[f"{project_name}_config.yaml"] = yaml.dump(config, indent=4) 

224 FilesManager.save_zip_file(content_to_zip, file_path=zip_filepath, force=force) 

225 

226 MessageHandler.print_success(f"Project was saved successfully -> {zip_filepath}") 

227 return None 

228 

229 

230def _save_unzipped_pep( 

231 project_dict: dict, folder_path: str, force: bool = False 

232) -> None: 

233 """ 

234 Save unzipped project to specified folder 

235 

236 :param project_dict: raw pep project 

237 :param folder_path: path to save project 

238 :param force: overwrite project if exists 

239 :return: None 

240 """ 

241 

242 def full_path(fn: str) -> str: 

243 return os.path.join(folder_path, fn) 

244 

245 project_name = project_dict[CONFIG_KEY][NAME_KEY] 

246 sample_table_filename = "sample_table.csv" 

247 yaml_full_path = full_path(f"{project_name}_config.yaml") 

248 sample_full_path = full_path(sample_table_filename) 

249 if not force: 

250 extant = [p for p in [yaml_full_path, sample_full_path] if os.path.isfile(p)] 

251 if extant: 

252 raise PEPExistsError(f"{len(extant)} file(s) exist(s): {', '.join(extant)}") 

253 

254 config_dict = project_dict.get(CONFIG_KEY) 

255 config_dict[NAME_KEY] = project_name 

256 config_dict[DESC_KEY] = project_dict[CONFIG_KEY][DESC_KEY] 

257 config_dict["sample_table"] = sample_table_filename 

258 

259 sample_pandas = pd.DataFrame(project_dict.get(SAMPLE_RAW_DICT_KEY, {})) 

260 

261 subsample_list = [ 

262 pd.DataFrame(sub_a) for sub_a in project_dict.get(SUBSAMPLE_RAW_LIST_KEY) or [] 

263 ] 

264 

265 filenames = [] 

266 for idx, subsample in enumerate(subsample_list): 

267 fn = f"subsample_table{idx + 1}.csv" 

268 filenames.append(fn) 

269 FilesManager.save_pandas(subsample, full_path(fn), not_force=False) 

270 config_dict["subsample_table"] = filenames 

271 

272 FilesManager.save_yaml(config_dict, yaml_full_path, not_force=False) 

273 FilesManager.save_pandas(sample_pandas, sample_full_path, not_force=False) 

274 

275 if config_dict.get("subsample_table"): 

276 for number, subsample in enumerate(subsample_list): 

277 FilesManager.save_pandas( 

278 subsample, 

279 os.path.join(folder_path, config_dict["subsample_table"][number]), 

280 not_force=False, 

281 ) 

282 

283 MessageHandler.print_success(f"Project was saved successfully -> {folder_path}") 

284 return None 

285 

286 

287def save_pep( 

288 project: Union[dict, peppy.Project], 

289 reg_path: str = None, 

290 force: bool = False, 

291 project_path: Optional[str] = None, 

292 zip: bool = False, 

293) -> None: 

294 """ 

295 Save project locally. 

296 

297 :param dict project: PEP dictionary (raw project) 

298 :param str reg_path: Project registry path in PEPhub (e.g. databio/base:default). If not provided, 

299 folder will be created with just project name. 

300 :param bool force: overwrite project if exists 

301 :param str project_path: Path where project will be saved. By default, it will be saved in current directory. 

302 :param bool zip: If True, save project as zip file 

303 :return: None 

304 """ 

305 if isinstance(project, peppy.Project): 

306 project = project.to_dict(extended=True, orient="records") 

307 

308 project = ProjectDict(**project).model_dump(by_alias=True) 

309 

310 if not project_path: 

311 project_path = os.getcwd() 

312 

313 if reg_path: 

314 file_name = _build_filename(RegistryPath(**parse_registry_path(reg_path))) 

315 else: 

316 file_name = project[CONFIG_KEY][NAME_KEY] 

317 

318 if zip: 

319 _save_zip_pep( 

320 project, 

321 zip_filepath=f"{os.path.join(project_path, file_name)}.zip", 

322 force=force, 

323 ) 

324 return None 

325 

326 folder_path = FilesManager.create_project_folder( 

327 parent_path=project_path, folder_name=file_name 

328 ) 

329 _save_unzipped_pep(project, folder_path, force=force) 

330 

331 

332def open_schema(file_path: Union[str, Path]) -> dict: 

333 """ 

334 Open schema file that are saved in yaml or json format. 

335 

336 :param file_path: path to the schema file 

337 

338 :raises: FileNotFoundError - if file doesn't exist 

339 :return: file object in dict format 

340 """ 

341 if isinstance(file_path, str): 

342 file_path = Path(file_path) 

343 

344 if not file_path.is_file(): 

345 raise FileNotFoundError( 

346 f"Provided schema file doesn't exist. File path: `{str(file_path)}`" 

347 ) 

348 

349 if file_path.suffix == ".yaml" or file_path.suffix == ".yml": 

350 with open(file_path, "r") as file: 

351 data = yaml.safe_load(file) 

352 

353 elif file_path.suffix == ".json": 

354 with open(file_path, "r") as file: 

355 data = json.load(file) 

356 else: 

357 raise BasePephubclientException( 

358 f"Incorrect file format provided: '{file_path.suffix}'. " 

359 "Only yaml and json formats are supported." 

360 ) 

361 

362 return data 

363 

364 

365def save_schema( 

366 file_path: Union[str, Path], 

367 schema_obj: dict, 

368 format: Literal["json", "yaml"] = "yaml", 

369) -> None: 

370 """ 

371 Save dict object as file in json or yaml format 

372 

373 :param file_path: path to the file 

374 :param schema_obj: content to be saved in the file 

375 :param format: Format in which file should be saved on disc. Default: yaml 

376 

377 :return: File path. 

378 """ 

379 

380 if format == "yaml": 

381 schema_obj = yaml.dump(schema_obj) 

382 

383 elif format == "json": 

384 schema_obj = json.dumps(schema_obj, indent=4) 

385 else: 

386 raise BasePephubclientException(f"Incorrect format provided: '{format}'") 

387 

388 with open(file_path, "w") as file: 

389 file.write(schema_obj) 

390 

391 

392def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: 

393 """ 

394 Convert schema path to namespace, name 

395 

396 :param schema_path: schema path that has structure: "namespace/name.yaml" 

397 :return: tuple(namespace, name, version) 

398 """ 

399 if "/" in schema_path: 

400 namespace, name_tag = schema_path.split("/") 

401 if ":" in name_tag: 

402 name, version = name_tag.split(":") 

403 return namespace, name, version 

404 

405 return namespace, name_tag, "latest" 

406 raise BasePephubclientException(f"Error in: '{schema_path}'")