Coverage for scripts/get_wiki.py: 0%

1import json 

2import logging 

3import re 

4 

5import requests 

6from bs4 import BeautifulSoup 

7 

8 

9def equals_to_dict(pairs: list[str]) -> dict[str, str]: 

10 """Convert a string in the format "key=value" into a dictionary.""" 

11 return { 

12 key.strip(): value.strip() for key, value in (pair.split("=") for pair in pairs) 

13 } 

14 

15 

16def main() -> None: 

17 """Parse the OSM wiki and return a mapping of Overture categories to OSM tags.""" 

18 tag_mapping = parse_wiki() 

19 

20 print("Writing tags to scripts/tags.json...") 

21 with open("scripts/tags.json", mode="w", encoding="utf-8") as f: 

22 f.write(json.dumps(tag_mapping, indent=4)) 

23 

24 

25def parse_wiki() -> dict: 

26 """Parse the OSM wiki and return a mapping of Overture categories to OSM tags.""" 

27 table = {} 

28 print("Getting tags from OSM wiki...") 

29 data = requests.get( 

30 "https://wiki.openstreetmap.org/w/api.php", 

31 {"action": "parse", "page": "Overture_categories", "format": "json"}, 

32 timeout=10, 

33 ).json() 

34 soup = BeautifulSoup(data["parse"]["text"]["*"], "html.parser") 

35 print("Parsing tags from OSM wiki...") 

36 a = soup.find("table") 

37 if a: 

38 for row in list(a.find_all("tr")): 

39 overture = row.find("td") 

40 overture_tag = "" 

41 if overture: 

42 overture_tag = overture.text 

43 

44 osm = row.find_all("tt") 

45 osm_tags = {} 

46 if osm: 

47 osm_tags = equals_to_dict([i.text for i in osm]) 

48 

49 if overture_tag: 

50 table[overture_tag] = osm_tags 

51 

52 return table 

53 

54 

55def parse_tags(wiki_str: str) -> dict: 

56 """Parse OSM tags from the wiki.""" 

57 tags = {} 

58 print(wiki_str) 

59 for tag in wiki_str.split(" and "): 

60 if m := re.match(r"^\s*([-_:\w]+)=([-_:\w]+)\s*$", tag): 

61 tags[m.group(1)] = m.group(2) 

62 else: 

63 logging.error(f"Failed to parse '{tag}' from '{wiki_str}'") 

64 

65 return tags 

66 

67 

68if __name__ == "__main__": 

69 main()