utils.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. __filename__ = "utils.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import os
  9. import datetime
  10. import commentjson
  11. def getStatusNumber() -> (str,str):
  12. """Returns the status number and published date
  13. """
  14. currTime=datetime.datetime.utcnow()
  15. daysSinceEpoch=(currTime - datetime.datetime(1970,1,1)).days
  16. # status is the number of seconds since epoch
  17. statusNumber=str(((daysSinceEpoch*24*60*60) + (currTime.hour*60*60) + (currTime.minute*60) + currTime.second)*1000000 + currTime.microsecond)
  18. published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  19. return statusNumber,published
  20. def isEvil(domain: str) -> bool:
  21. # https://www.youtube.com/watch?v=5qw1hcevmdU
  22. evilDomains=['gab.com','gabfed.com','spinster.xyz','djitter.com']
  23. for concentratedEvil in evilDomains:
  24. if domain.endswith(concentratedEvil):
  25. return True
  26. return False
  27. def createPersonDir(nickname: str,domain: str,baseDir: str,dirname: str) -> str:
  28. """Create a directory for a person
  29. """
  30. handle=nickname+'@'+domain
  31. if not os.path.isdir(baseDir+'/accounts/'+handle):
  32. os.mkdir(baseDir+'/accounts/'+handle)
  33. boxDir=baseDir+'/accounts/'+handle+'/'+dirname
  34. if not os.path.isdir(boxDir):
  35. os.mkdir(boxDir)
  36. return boxDir
  37. def createOutboxDir(nickname: str,domain: str,baseDir: str) -> str:
  38. """Create an outbox for a person
  39. """
  40. return createPersonDir(nickname,domain,baseDir,'outbox')
  41. def createInboxQueueDir(nickname: str,domain: str,baseDir: str) -> str:
  42. """Create an inbox queue and returns the feed filename and directory
  43. """
  44. return createPersonDir(nickname,domain,baseDir,'queue')
  45. def domainPermitted(domain: str, federationList: []):
  46. if len(federationList)==0:
  47. return True
  48. if ':' in domain:
  49. domain=domain.split(':')[0]
  50. if domain in federationList:
  51. return True
  52. return False
  53. def urlPermitted(url: str, federationList: [],capability: str):
  54. if isEvil(url):
  55. return False
  56. if len(federationList)==0:
  57. return True
  58. for domain in federationList:
  59. if domain in url:
  60. return True
  61. return False
  62. def getDisplayName(actor: str,personCache: {}) -> str:
  63. """Returns the display name for the given actor
  64. """
  65. if '/statuses/' in actor:
  66. actor=actor.split('/statuses/')[0]
  67. if not personCache.get(actor):
  68. return None
  69. if personCache[actor].get('actor'):
  70. if personCache[actor]['actor'].get('name'):
  71. return personCache[actor]['actor']['name']
  72. return None
  73. def getNicknameFromActor(actor: str) -> str:
  74. """Returns the nickname from an actor url
  75. """
  76. if '/users/' not in actor:
  77. if '/profile/' in actor:
  78. nickStr=actor.split('/profile/')[1].replace('@','')
  79. if '/' not in nickStr:
  80. return nickStr
  81. else:
  82. return nickStr.split('/')[0]
  83. # https://domain/@nick
  84. if '/@' in actor:
  85. nickStr=actor.split('/@')[1]
  86. if '/' in nickStr:
  87. nickStr=nickStr.split('/')[0]
  88. return nickStr
  89. return None
  90. nickStr=actor.split('/users/')[1].replace('@','')
  91. if '/' not in nickStr:
  92. return nickStr
  93. else:
  94. return nickStr.split('/')[0]
  95. def getDomainFromActor(actor: str) -> (str,int):
  96. """Returns the domain name from an actor url
  97. """
  98. port=None
  99. if '/profile/' in actor:
  100. domain = actor.split('/profile/')[0].replace('https://','').replace('http://','').replace('dat://','')
  101. else:
  102. if '/users/' not in actor:
  103. domain = actor.replace('https://','').replace('http://','').replace('dat://','')
  104. if '/' in actor:
  105. domain=domain.split('/')[0]
  106. else:
  107. domain = actor.split('/users/')[0].replace('https://','').replace('http://','').replace('dat://','')
  108. if ':' in domain:
  109. port=int(domain.split(':')[1])
  110. domain=domain.split(':')[0]
  111. return domain,port
  112. def followPerson(baseDir: str,nickname: str, domain: str, \
  113. followNickname: str, followDomain: str, \
  114. federationList: [],debug: bool, \
  115. followFile='following.txt') -> bool:
  116. """Adds a person to the follow list
  117. """
  118. if not domainPermitted(followDomain.lower().replace('\n',''), \
  119. federationList):
  120. if debug:
  121. print('DEBUG: follow of domain '+followDomain+' not permitted')
  122. return False
  123. if debug:
  124. print('DEBUG: follow of domain '+followDomain)
  125. if ':' in domain:
  126. handle=nickname+'@'+domain.split(':')[0].lower()
  127. else:
  128. handle=nickname+'@'+domain.lower()
  129. if ':' in followDomain:
  130. handleToFollow=followNickname+'@'+followDomain.split(':')[0].lower()
  131. else:
  132. handleToFollow=followNickname+'@'+followDomain.lower()
  133. if not os.path.isdir(baseDir+'/accounts'):
  134. os.mkdir(baseDir+'/accounts')
  135. if not os.path.isdir(baseDir+'/accounts/'+handle):
  136. os.mkdir(baseDir+'/accounts/'+handle)
  137. filename=baseDir+'/accounts/'+handle+'/'+followFile
  138. if os.path.isfile(filename):
  139. if handleToFollow in open(filename).read():
  140. if debug:
  141. print('DEBUG: follow already exists')
  142. return True
  143. with open(filename, "a") as followfile:
  144. followfile.write(followNickname+'@'+followDomain+'\n')
  145. if debug:
  146. print('DEBUG: follow added')
  147. return True
  148. if debug:
  149. print('DEBUG: creating new following file')
  150. with open(filename, "w") as followfile:
  151. followfile.write(followNickname+'@'+followDomain+'\n')
  152. return True
  153. def locatePost(baseDir: str,nickname: str,domain: str,postUrl: str,replies=False) -> str:
  154. """Returns the filename for the given status post url
  155. """
  156. if not replies:
  157. extension='json'
  158. else:
  159. extension='replies'
  160. # if this post in the shared inbox?
  161. handle='inbox@'+domain
  162. boxName='inbox'
  163. postUrl=postUrl.replace('/','#')
  164. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
  165. if not os.path.isfile(postFilename):
  166. boxName='outbox'
  167. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
  168. if not os.path.isfile(postFilename):
  169. # if this post in the inbox of the person?
  170. boxName='inbox'
  171. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
  172. if not os.path.isfile(postFilename):
  173. postFilename=baseDir+'/cache/announce/'+nickname+'/'+postUrl+'.'+extension
  174. if not os.path.isfile(postFilename):
  175. postFilename=None
  176. return postFilename
  177. def removeAttachment(baseDir: str,httpPrefix: str,domain: str,postJson: {}):
  178. if not postJson.get('attachment'):
  179. return
  180. if not postJson['attachment'][0].get('url'):
  181. return
  182. if port:
  183. if port!=80 and port!=443:
  184. if ':' not in domain:
  185. domain=domain+':'+str(port)
  186. attachmentUrl=postJson['attachment'][0]['url']
  187. if not attachmentUrl:
  188. return
  189. mediaFilename=baseDir+'/'+attachmentUrl.replace(httpPrefix+'://'+domain+'/','')
  190. if os.path.isfile(mediaFilename):
  191. os.remove(mediaFilename)
  192. postJson['attachment']=[]
  193. def removeModerationPostFromIndex(baseDir: str,postUrl: str,debug: bool) -> None:
  194. """Removes a url from the moderation index
  195. """
  196. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  197. if not os.path.isfile(moderationIndexFile):
  198. return
  199. postId=postUrl.replace('/activity','')
  200. if postId in open(moderationIndexFile).read():
  201. with open(moderationIndexFile, "r") as f:
  202. lines = f.readlines()
  203. with open(moderationIndexFile, "w+") as f:
  204. for line in lines:
  205. if line.strip("\n") != postId:
  206. f.write(line)
  207. else:
  208. if debug:
  209. print('DEBUG: removed '+postId+' from moderation index')
  210. def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilename: str,debug: bool):
  211. """Recursively deletes a post and its replies and attachments
  212. """
  213. with open(postFilename, 'r') as fp:
  214. postJsonObject=commentjson.load(fp)
  215. # remove any attachment
  216. removeAttachment(baseDir,httpPrefix,domain,postJsonObject)
  217. # remove from moderation index file
  218. if postJsonObject.get('moderationStatus'):
  219. if postJsonObject.get('object'):
  220. if isinstance(postJsonObject['object'], dict):
  221. if postJsonObject['object'].get('id'):
  222. postId=postJsonObject['object']['id'].replace('/activity','')
  223. removeModerationPostFromIndex(baseDir,postId,debug)
  224. # remove any hashtags index entries
  225. removeHashtagIndex=False
  226. if postJsonObject.get('object'):
  227. if isinstance(postJsonObject['object'], dict):
  228. if postJsonObject['object'].get('content'):
  229. if '#' in postJsonObject['object']['content']:
  230. removeHashtagIndex=True
  231. if removeHashtagIndex:
  232. if postJsonObject['object'].get('id') and postJsonObject['object'].get('tag'):
  233. # get the id of the post
  234. postId=postJsonObject['object']['id'].replace('/activity','')
  235. for tag in postJsonObject['object']['tag']:
  236. if tag['type']!='Hashtag':
  237. continue
  238. # find the index file for this tag
  239. tagIndexFilename=baseDir+'/tags/'+tag['name'][1:]+'.txt'
  240. if not os.path.isfile(tagIndexFilename):
  241. continue
  242. # remove postId from the tag index file
  243. with open(tagIndexFilename, "r") as f:
  244. lines = f.readlines()
  245. with open(tagIndexFilename, "w+") as f:
  246. for line in lines:
  247. if line.strip("\n") != postId:
  248. f.write(line)
  249. # remove any replies
  250. repliesFilename=postFilename.replace('.json','.replies')
  251. if os.path.isfile(repliesFilename):
  252. if debug:
  253. print('DEBUG: removing replies to '+postFilename)
  254. with open(repliesFilename,'r') as f:
  255. for replyId in f:
  256. replyFile=locatePost(baseDir,nickname,domain,replyId)
  257. if replyFile:
  258. if os.path.isfile(replyFile):
  259. deletePost(baseDir,nickname,domain,replyFile,debug)
  260. # remove the replies file
  261. os.remove(repliesFilename)
  262. # finally, remove the post itself
  263. os.remove(postFilename)
  264. def validNickname(domain: str,nickname: str) -> bool:
  265. forbiddenChars=['.',' ','/','?',':',';','@']
  266. for c in forbiddenChars:
  267. if c in nickname:
  268. return False
  269. if nickname==domain:
  270. return False
  271. reservedNames=['inbox','dm','outbox','following','public','followers','capabilities']
  272. if nickname in reservedNames:
  273. return False
  274. return True
  275. def noOfAccounts(baseDir: str) -> bool:
  276. """Returns the number of accounts on the system
  277. """
  278. accountCtr=0
  279. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  280. for account in dirs:
  281. if '@' in account:
  282. if not account.startswith('inbox'):
  283. accountCtr+=1
  284. return accountCtr
  285. def isPublicPost(postJsonObject: {}) -> bool:
  286. """Returns true if the given post is public
  287. """
  288. if not postJsonObject.get('type'):
  289. return False
  290. if postJsonObject['type']!='Create':
  291. return False
  292. if not postJsonObject.get('object'):
  293. return False
  294. if not isinstance(postJsonObject['object'], dict):
  295. return False
  296. if not postJsonObject['object'].get('to'):
  297. return False
  298. for recipient in postJsonObject['object']['to']:
  299. if recipient.endswith('#Public'):
  300. return True
  301. return False