utils.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. __filename__ = "utils.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "0.0.1"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import os
  9. import datetime
  10. import commentjson
  11. def getStatusNumber() -> (str,str):
  12. """Returns the status number and published date
  13. """
  14. currTime=datetime.datetime.utcnow()
  15. daysSinceEpoch=(currTime - datetime.datetime(1970,1,1)).days
  16. # status is the number of seconds since epoch
  17. statusNumber=str(((daysSinceEpoch*24*60*60) + (currTime.hour*60*60) + (currTime.minute*60) + currTime.second)*1000000 + currTime.microsecond)
  18. published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  19. conversationDate=currTime.strftime("%Y-%m-%d")
  20. return statusNumber,published
  21. def createPersonDir(nickname: str,domain: str,baseDir: str,dirname: str) -> str:
  22. """Create a directory for a person
  23. """
  24. handle=nickname+'@'+domain
  25. if not os.path.isdir(baseDir+'/accounts/'+handle):
  26. os.mkdir(baseDir+'/accounts/'+handle)
  27. boxDir=baseDir+'/accounts/'+handle+'/'+dirname
  28. if not os.path.isdir(boxDir):
  29. os.mkdir(boxDir)
  30. return boxDir
  31. def createOutboxDir(nickname: str,domain: str,baseDir: str) -> str:
  32. """Create an outbox for a person
  33. """
  34. return createPersonDir(nickname,domain,baseDir,'outbox')
  35. def createInboxQueueDir(nickname: str,domain: str,baseDir: str) -> str:
  36. """Create an inbox queue and returns the feed filename and directory
  37. """
  38. return createPersonDir(nickname,domain,baseDir,'queue')
  39. def domainPermitted(domain: str, federationList: []):
  40. if len(federationList)==0:
  41. return True
  42. if ':' in domain:
  43. domain=domain.split(':')[0]
  44. if domain in federationList:
  45. return True
  46. return False
  47. def urlPermitted(url: str, federationList: [],capability: str):
  48. if url.endswith('gab.com') or url.endswith('gabfed.com'):
  49. return False
  50. if len(federationList)==0:
  51. return True
  52. for domain in federationList:
  53. if domain in url:
  54. return True
  55. return False
  56. def getNicknameFromActor(actor: str) -> str:
  57. """Returns the nickname from an actor url
  58. """
  59. if '/users/' not in actor:
  60. return None
  61. nickStr=actor.split('/users/')[1].replace('@','')
  62. if '/' not in nickStr:
  63. return nickStr
  64. else:
  65. return nickStr.split('/')[0]
  66. def getDomainFromActor(actor: str) -> (str,int):
  67. """Returns the domain name from an actor url
  68. """
  69. port=None
  70. if '/users/' not in actor:
  71. domain = actor.replace('https://','').replace('http://','').replace('dat://','')
  72. else:
  73. domain = actor.split('/users/')[0].replace('https://','').replace('http://','').replace('dat://','')
  74. if ':' in domain:
  75. port=int(domain.split(':')[1])
  76. domain=domain.split(':')[0]
  77. return domain,port
  78. def followPerson(baseDir: str,nickname: str, domain: str, \
  79. followNickname: str, followDomain: str, \
  80. federationList: [],debug: bool, \
  81. followFile='following.txt') -> bool:
  82. """Adds a person to the follow list
  83. """
  84. if not domainPermitted(followDomain.lower().replace('\n',''), \
  85. federationList):
  86. if debug:
  87. print('DEBUG: follow of domain '+followDomain+' not permitted')
  88. return False
  89. if debug:
  90. print('DEBUG: follow of domain '+followDomain)
  91. if ':' in domain:
  92. handle=nickname+'@'+domain.split(':')[0].lower()
  93. else:
  94. handle=nickname+'@'+domain.lower()
  95. if ':' in followDomain:
  96. handleToFollow=followNickname+'@'+followDomain.split(':')[0].lower()
  97. else:
  98. handleToFollow=followNickname+'@'+followDomain.lower()
  99. if not os.path.isdir(baseDir+'/accounts'):
  100. os.mkdir(baseDir+'/accounts')
  101. if not os.path.isdir(baseDir+'/accounts/'+handle):
  102. os.mkdir(baseDir+'/accounts/'+handle)
  103. filename=baseDir+'/accounts/'+handle+'/'+followFile
  104. if os.path.isfile(filename):
  105. if handleToFollow in open(filename).read():
  106. if debug:
  107. print('DEBUG: follow already exists')
  108. return True
  109. with open(filename, "a") as followfile:
  110. followfile.write(followNickname+'@'+followDomain+'\n')
  111. if debug:
  112. print('DEBUG: follow added')
  113. return True
  114. if debug:
  115. print('DEBUG: creating new following file')
  116. with open(filename, "w") as followfile:
  117. followfile.write(followNickname+'@'+followDomain+'\n')
  118. return True
  119. def locatePost(baseDir: str,nickname: str,domain: str,postUrl: str,replies=False) -> str:
  120. """Returns the filename for the given status post url
  121. """
  122. if not replies:
  123. extension='json'
  124. else:
  125. extension='replies'
  126. # if this post in the shared inbox?
  127. handle='inbox@'+domain
  128. boxName='inbox'
  129. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl.replace('/','#')+'.'+extension
  130. if not os.path.isfile(postFilename):
  131. boxName='outbox'
  132. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl.replace('/','#')+'.'+extension
  133. if not os.path.isfile(postFilename):
  134. # if this post in the inbox of the person?
  135. boxName='inbox'
  136. postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl.replace('/','#')+'.'+extension
  137. if not os.path.isfile(postFilename):
  138. postFilename=None
  139. return postFilename
  140. def removeAttachment(baseDir: str,httpPrefix: str,domain: str,postJson: {}):
  141. if not postJson.get('attachment'):
  142. return
  143. if not postJson['attachment'][0].get('url'):
  144. return
  145. if port:
  146. if port!=80 and port!=443:
  147. if ':' not in domain:
  148. domain=domain+':'+str(port)
  149. attachmentUrl=postJson['attachment'][0]['url']
  150. if not attachmentUrl:
  151. return
  152. mediaFilename=baseDir+'/'+attachmentUrl.replace(httpPrefix+'://'+domain+'/','')
  153. if os.path.isfile(mediaFilename):
  154. os.remove(mediaFilename)
  155. postJson['attachment']=[]
  156. def removeModerationPostFromIndex(baseDir: str,postUrl: str,debug: bool) -> None:
  157. """Removes a url from the moderation index
  158. """
  159. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  160. if not os.path.isfile(moderationIndexFile):
  161. return
  162. postId=postUrl.replace('/activity','')
  163. if postId in open(moderationIndexFile).read():
  164. with open(moderationIndexFile, "r") as f:
  165. lines = f.readlines()
  166. with open(moderationIndexFile, "w+") as f:
  167. for line in lines:
  168. if line.strip("\n") != postId:
  169. f.write(line)
  170. else:
  171. if debug:
  172. print('DEBUG: removed '+postId+' from moderation index')
  173. def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilename: str,debug: bool):
  174. """Recursively deletes a post and its replies and attachments
  175. """
  176. with open(postFilename, 'r') as fp:
  177. postJsonObject=commentjson.load(fp)
  178. # remove any attachment
  179. removeAttachment(baseDir,httpPrefix,domain,postJsonObject)
  180. # remove from moderation index file
  181. if postJsonObject.get('moderationStatus'):
  182. if postJsonObject.get('object'):
  183. if isinstance(postJsonObject['object'], dict):
  184. if postJsonObject['object'].get('id'):
  185. postId=postJsonObject['object']['id'].replace('/activity','')
  186. removeModerationPostFromIndex(baseDir,postId,debug)
  187. # remove any hashtags index entries
  188. removeHashtagIndex=False
  189. if postJsonObject.get('object'):
  190. if isinstance(postJsonObject['object'], dict):
  191. if postJsonObject['object'].get('content'):
  192. if '#' in postJsonObject['object']['content']:
  193. removeHashtagIndex=True
  194. if removeHashtagIndex:
  195. if postJsonObject['object'].get('id') and postJsonObject['object'].get('tag'):
  196. # get the id of the post
  197. postId=postJsonObject['object']['id'].replace('/activity','')
  198. for tag in postJsonObject['object']['tag']:
  199. if tag['type']!='Hashtag':
  200. continue
  201. # find the index file for this tag
  202. tagIndexFilename=baseDir+'/tags/'+tag['name'][1:]+'.txt'
  203. if not os.path.isfile(tagIndexFilename):
  204. continue
  205. # remove postId from the tag index file
  206. with open(tagIndexFilename, "r") as f:
  207. lines = f.readlines()
  208. with open(tagIndexFilename, "w+") as f:
  209. for line in lines:
  210. if line.strip("\n") != postId:
  211. f.write(line)
  212. # remove any replies
  213. repliesFilename=postFilename.replace('.json','.replies')
  214. if os.path.isfile(repliesFilename):
  215. if debug:
  216. print('DEBUG: removing replies to '+postFilename)
  217. with open(repliesFilename,'r') as f:
  218. for replyId in f:
  219. replyFile=locatePost(baseDir,nickname,domain,replyId)
  220. if replyFile:
  221. if os.path.isfile(replyFile):
  222. deletePost(baseDir,nickname,domain,replyFile,debug)
  223. # remove the replies file
  224. os.remove(repliesFilename)
  225. # finally, remove the post itself
  226. os.remove(postFilename)
  227. def validNickname(nickname: str) -> bool:
  228. forbiddenChars=['.',' ','/','?',':',';','@']
  229. for c in forbiddenChars:
  230. if c in nickname:
  231. return False
  232. reservedNames=['inbox','outbox','following','followers','capabilities']
  233. if nickname in reservedNames:
  234. return False
  235. return True
  236. def noOfAccounts(baseDir: str) -> bool:
  237. """Returns the number of accounts on the system
  238. """
  239. accountCtr=0
  240. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  241. for account in dirs:
  242. if '@' in account:
  243. if not account.startswith('inbox'):
  244. accountCtr+=1
  245. return accountCtr
  246. def isPublicPost(postJsonObject: {}) -> bool:
  247. """Returns true if the given post is public
  248. """
  249. if not postJsonObject.get('type'):
  250. return False
  251. if postJsonObject['type']!='Create':
  252. return False
  253. if not postJsonObject.get('object'):
  254. return False
  255. if not isinstance(postJsonObject['object'], dict):
  256. return False
  257. if not postJsonObject['object'].get('to'):
  258. return False
  259. for recipient in postJsonObject['object']['to']:
  260. if recipient.endswith('#Public'):
  261. return True
  262. return False