posts.py 83 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import requests
  9. import json
  10. import commentjson
  11. import html
  12. import datetime
  13. import os
  14. import shutil
  15. import threading
  16. import sys
  17. import trace
  18. import time
  19. from collections import OrderedDict
  20. from threads import threadWithTrace
  21. from cache import storePersonInCache
  22. from cache import getPersonFromCache
  23. from cache import expirePersonCache
  24. from pprint import pprint
  25. from random import randint
  26. from session import createSession
  27. from session import getJson
  28. from session import postJsonString
  29. from session import postImage
  30. from webfinger import webfingerHandle
  31. from httpsig import createSignedHeader
  32. from utils import getStatusNumber
  33. from utils import createPersonDir
  34. from utils import urlPermitted
  35. from utils import getNicknameFromActor
  36. from utils import getDomainFromActor
  37. from utils import deletePost
  38. from utils import validNickname
  39. from capabilities import getOcapFilename
  40. from capabilities import capabilitiesUpdate
  41. from media import attachMedia
  42. from content import addHtmlTags
  43. from auth import createBasicAuthHeader
  44. from config import getConfigParam
  45. try:
  46. from BeautifulSoup import BeautifulSoup
  47. except ImportError:
  48. from bs4 import BeautifulSoup
  49. def isModerator(baseDir: str,nickname: str) -> bool:
  50. """Returns true if the given nickname is a moderator
  51. """
  52. moderatorsFile=baseDir+'/accounts/moderators.txt'
  53. if not os.path.isfile(moderatorsFile):
  54. if getConfigParam(baseDir,'admin')==nickname:
  55. return True
  56. return False
  57. with open(moderatorsFile, "r") as f:
  58. lines = f.readlines()
  59. if len(lines)==0:
  60. if getConfigParam(baseDir,'admin')==nickname:
  61. return True
  62. for moderator in lines:
  63. moderator=moderator.strip('\n')
  64. if moderator==nickname:
  65. return True
  66. return False
  67. def noOfFollowersOnDomain(baseDir: str,handle: str, \
  68. domain: str, followFile='followers.txt') -> int:
  69. """Returns the number of followers of the given handle from the given domain
  70. """
  71. filename=baseDir+'/accounts/'+handle+'/'+followFile
  72. if not os.path.isfile(filename):
  73. return 0
  74. ctr=0
  75. with open(filename, "r") as followersFilename:
  76. for followerHandle in followersFilename:
  77. if '@' in followerHandle:
  78. followerDomain= \
  79. followerHandle.split('@')[1].replace('\n','')
  80. if domain==followerDomain:
  81. ctr+=1
  82. return ctr
  83. def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
  84. debug=False):
  85. """Returns the public or private key of a person
  86. """
  87. handle=nickname+'@'+domain
  88. keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
  89. if not os.path.isfile(keyFilename):
  90. if debug:
  91. print('DEBUG: private key file not found: '+keyFilename)
  92. return ''
  93. keyPem=''
  94. with open(keyFilename, "r") as pemFile:
  95. keyPem=pemFile.read()
  96. if len(keyPem)<20:
  97. if debug:
  98. print('DEBUG: private key was too short: '+keyPem)
  99. return ''
  100. return keyPem
  101. def cleanHtml(rawHtml: str) -> str:
  102. text = BeautifulSoup(rawHtml, 'html.parser').get_text()
  103. return html.unescape(text)
  104. def getUserUrl(wfRequest) -> str:
  105. if wfRequest.get('links'):
  106. for link in wfRequest['links']:
  107. if link.get('type') and link.get('href'):
  108. if link['type'] == 'application/activity+json':
  109. if '/users/' in link['href'] or '/profile/' in link['href']:
  110. return link['href']
  111. else:
  112. print('Webfinger activity+json does not contain a valid actor')
  113. print(link['href'])
  114. return None
  115. def parseUserFeed(session,feedUrl: str,asHeader: {}, \
  116. projectVersion: str,httpPrefix: str,domain: str) -> None:
  117. feedJson = getJson(session,feedUrl,asHeader,None, \
  118. projectVersion,httpPrefix,domain)
  119. if not feedJson:
  120. return
  121. if 'orderedItems' in feedJson:
  122. for item in feedJson['orderedItems']:
  123. yield item
  124. nextUrl = None
  125. if 'first' in feedJson:
  126. nextUrl = feedJson['first']
  127. elif 'next' in feedJson:
  128. nextUrl = feedJson['next']
  129. if nextUrl:
  130. if isinstance(nextUrl, str):
  131. userFeed=parseUserFeed(session,nextUrl,asHeader, \
  132. projectVersion,httpPrefix,domain)
  133. for item in userFeed:
  134. yield item
  135. elif isinstance(nextUrl, dict):
  136. userFeed=nextUrl
  137. if userFeed.get('orderedItems'):
  138. for item in userFeed['orderedItems']:
  139. yield item
  140. def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
  141. projectVersion: str,httpPrefix: str,domain: str, \
  142. boxName='inbox') -> (str,str,str,str,str,str,str,str):
  143. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  144. personUrl = getUserUrl(wfRequest)
  145. if not personUrl:
  146. return None,None,None,None,None,None,None,None
  147. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  148. if not personJson:
  149. personJson = getJson(session,personUrl,asHeader,None, \
  150. projectVersion,httpPrefix,domain)
  151. if not personJson:
  152. return None,None,None,None,None,None,None,None
  153. boxJson=None
  154. if not personJson.get(boxName):
  155. if personJson.get('endpoints'):
  156. if personJson['endpoints'].get(boxName):
  157. boxJson=personJson['endpoints'][boxName]
  158. else:
  159. boxJson=personJson[boxName]
  160. if not boxJson:
  161. return None,None,None,None,None,None,None,None
  162. personId=None
  163. if personJson.get('id'):
  164. personId=personJson['id']
  165. pubKeyId=None
  166. pubKey=None
  167. if personJson.get('publicKey'):
  168. if personJson['publicKey'].get('id'):
  169. pubKeyId=personJson['publicKey']['id']
  170. if personJson['publicKey'].get('publicKeyPem'):
  171. pubKey=personJson['publicKey']['publicKeyPem']
  172. sharedInbox=None
  173. if personJson.get('sharedInbox'):
  174. sharedInbox=personJson['sharedInbox']
  175. else:
  176. if personJson.get('endpoints'):
  177. if personJson['endpoints'].get('sharedInbox'):
  178. sharedInbox=personJson['endpoints']['sharedInbox']
  179. capabilityAcquisition=None
  180. if personJson.get('capabilityAcquisitionEndpoint'):
  181. capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
  182. avatarUrl=None
  183. if personJson.get('icon'):
  184. if personJson['icon'].get('url'):
  185. avatarUrl=personJson['icon']['url']
  186. displayName=None
  187. if personJson.get('name'):
  188. displayName=personJson['name']
  189. storePersonInCache(baseDir,personUrl,personJson,personCache)
  190. return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
  191. def getPosts(session,outboxUrl: str,maxPosts: int, \
  192. maxMentions: int, \
  193. maxEmoji: int,maxAttachments: int, \
  194. federationList: [], \
  195. personCache: {},raw: bool, \
  196. simple: bool,debug: bool, \
  197. projectVersion: str,httpPrefix: str,domain: str) -> {}:
  198. """Gets public posts from an outbox
  199. """
  200. personPosts={}
  201. if not outboxUrl:
  202. return personPosts
  203. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  204. if raw:
  205. result = []
  206. i = 0
  207. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  208. projectVersion,httpPrefix,domain)
  209. for item in userFeed:
  210. result.append(item)
  211. i += 1
  212. if i == maxPosts:
  213. break
  214. pprint(result)
  215. return None
  216. i = 0
  217. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  218. projectVersion,httpPrefix,domain)
  219. for item in userFeed:
  220. if not item.get('id'):
  221. if debug:
  222. print('No id')
  223. continue
  224. if not item.get('type'):
  225. if debug:
  226. print('No type')
  227. continue
  228. if item['type'] != 'Create':
  229. if debug:
  230. print('Not Create type')
  231. continue
  232. if not item.get('object'):
  233. if debug:
  234. print('No object')
  235. continue
  236. if not isinstance(item['object'], dict):
  237. if debug:
  238. print('item object is not a dict')
  239. continue
  240. if not item['object'].get('published'):
  241. if debug:
  242. print('No published attribute')
  243. continue
  244. #pprint(item)
  245. published = item['object']['published']
  246. if not personPosts.get(item['id']):
  247. # check that this is a public post
  248. # #Public should appear in the "to" list
  249. if item['object'].get('to'):
  250. isPublic=False
  251. for recipient in item['object']['to']:
  252. if recipient.endswith('#Public'):
  253. isPublic=True
  254. break
  255. if not isPublic:
  256. continue
  257. content = item['object']['content'].replace('&apos;',"'")
  258. mentions=[]
  259. emoji={}
  260. if item['object'].get('tag'):
  261. for tagItem in item['object']['tag']:
  262. tagType=tagItem['type'].lower()
  263. if tagType=='emoji':
  264. if tagItem.get('name') and tagItem.get('icon'):
  265. if tagItem['icon'].get('url'):
  266. # No emoji from non-permitted domains
  267. if urlPermitted(tagItem['icon']['url'], \
  268. federationList, \
  269. "objects:read"):
  270. emojiName=tagItem['name']
  271. emojiIcon=tagItem['icon']['url']
  272. emoji[emojiName]=emojiIcon
  273. else:
  274. if debug:
  275. print('url not permitted '+tagItem['icon']['url'])
  276. if tagType=='mention':
  277. if tagItem.get('name'):
  278. if tagItem['name'] not in mentions:
  279. mentions.append(tagItem['name'])
  280. if len(mentions)>maxMentions:
  281. if debug:
  282. print('max mentions reached')
  283. continue
  284. if len(emoji)>maxEmoji:
  285. if debug:
  286. print('max emojis reached')
  287. continue
  288. summary = ''
  289. if item['object'].get('summary'):
  290. if item['object']['summary']:
  291. summary = item['object']['summary']
  292. inReplyTo = ''
  293. if item['object'].get('inReplyTo'):
  294. if item['object']['inReplyTo']:
  295. # No replies to non-permitted domains
  296. if not urlPermitted(item['object']['inReplyTo'], \
  297. federationList, \
  298. "objects:read"):
  299. if debug:
  300. print('url not permitted '+item['object']['inReplyTo'])
  301. continue
  302. inReplyTo = item['object']['inReplyTo']
  303. conversation = ''
  304. if item['object'].get('conversation'):
  305. if item['object']['conversation']:
  306. # no conversations originated in non-permitted domains
  307. if urlPermitted(item['object']['conversation'], \
  308. federationList,"objects:read"):
  309. conversation = item['object']['conversation']
  310. attachment = []
  311. if item['object'].get('attachment'):
  312. if item['object']['attachment']:
  313. for attach in item['object']['attachment']:
  314. if attach.get('name') and attach.get('url'):
  315. # no attachments from non-permitted domains
  316. if urlPermitted(attach['url'], \
  317. federationList, \
  318. "objects:read"):
  319. attachment.append([attach['name'],attach['url']])
  320. else:
  321. if debug:
  322. print('url not permitted '+attach['url'])
  323. sensitive = False
  324. if item['object'].get('sensitive'):
  325. sensitive = item['object']['sensitive']
  326. if simple:
  327. print(cleanHtml(content)+'\n')
  328. else:
  329. pprint(item)
  330. personPosts[item['id']] = {
  331. "sensitive": sensitive,
  332. "inreplyto": inReplyTo,
  333. "summary": summary,
  334. "html": content,
  335. "plaintext": cleanHtml(content),
  336. "attachment": attachment,
  337. "mentions": mentions,
  338. "emoji": emoji,
  339. "conversation": conversation
  340. }
  341. i += 1
  342. if i == maxPosts:
  343. break
  344. return personPosts
  345. def deleteAllPosts(baseDir: str, \
  346. nickname: str, domain: str,boxname: str) -> None:
  347. """Deletes all posts for a person from inbox or outbox
  348. """
  349. if boxname!='inbox' and boxname!='outbox':
  350. return
  351. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  352. for deleteFilename in os.listdir(boxDir):
  353. filePath = os.path.join(boxDir, deleteFilename)
  354. try:
  355. if os.path.isfile(filePath):
  356. os.unlink(filePath)
  357. elif os.path.isdir(filePath): shutil.rmtree(filePath)
  358. except Exception as e:
  359. print(e)
  360. def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
  361. nickname: str, domain: str,postJsonObject: {}, \
  362. boxname: str) -> str:
  363. """Saves the give json to the give box
  364. Returns the filename
  365. """
  366. if boxname!='inbox' and boxname!='outbox':
  367. return None
  368. originalDomain=domain
  369. if ':' in domain:
  370. domain=domain.split(':')[0]
  371. if not postId:
  372. statusNumber,published = getStatusNumber()
  373. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  374. postJsonObject['id']=postId+'/activity'
  375. if postJsonObject.get('object'):
  376. if isinstance(postJsonObject['object'], dict):
  377. postJsonObject['object']['id']=postId
  378. postJsonObject['object']['atomUri']=postId
  379. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  380. filename=boxDir+'/'+postId.replace('/','#')+'.json'
  381. with open(filename, 'w') as fp:
  382. commentjson.dump(postJsonObject, fp, indent=4, sort_keys=False)
  383. return filename
  384. def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
  385. """Writes the post url for hashtags to a file
  386. This allows posts for a hashtag to be quickly looked up
  387. """
  388. if tag['type']!='Hashtag':
  389. return
  390. # create hashtags directory
  391. tagsDir=baseDir+'/tags'
  392. if not os.path.isdir(tagsDir):
  393. os.mkdir(tagsDir)
  394. tagName=tag['name']
  395. tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
  396. tagFile=open(tagsFilename, "a+")
  397. if not tagFile:
  398. return
  399. tagFile.write(newPostId+'\n')
  400. tagFile.close()
  401. def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
  402. toUrl: str, ccUrl: str, httpPrefix: str, content: str, \
  403. followersOnly: bool, saveToFile: bool, clientToServer: bool, \
  404. attachImageFilename: str,mediaType: str,imageDescription: str, \
  405. useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
  406. inReplyToAtomUri=None, subject=None) -> {}:
  407. """Creates a message
  408. """
  409. mentionedRecipients= \
  410. getMentionedPeople(baseDir,httpPrefix,content,domain,False)
  411. tags=[]
  412. hashtagsDict={}
  413. if port:
  414. if port!=80 and port!=443:
  415. if ':' not in domain:
  416. domain=domain+':'+str(port)
  417. # convert content to html
  418. content= \
  419. addHtmlTags(baseDir,httpPrefix, \
  420. nickname,domain,content, \
  421. mentionedRecipients, \
  422. hashtagsDict)
  423. statusNumber,published = getStatusNumber()
  424. postTo='https://www.w3.org/ns/activitystreams#Public'
  425. postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  426. if followersOnly:
  427. postTo=postCC
  428. postCC=''
  429. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  430. sensitive=False
  431. summary=None
  432. if subject:
  433. summary=subject
  434. sensitive=True
  435. toRecipients=[]
  436. if toUrl:
  437. if not isinstance(toUrl, str):
  438. print('ERROR: toUrl is not a string')
  439. return None
  440. toRecipients=[toUrl]
  441. # who to send to
  442. if mentionedRecipients:
  443. for mention in mentionedRecipients:
  444. if mention not in toRecipients:
  445. toRecipients.append(mention)
  446. # create a list of hashtags
  447. # Only posts which are #Public are searchable by hashtag
  448. if hashtagsDict:
  449. isPublic=False
  450. for recipient in toRecipients:
  451. if recipient.endswith('#Public'):
  452. isPublic=True
  453. break
  454. for tagName,tag in hashtagsDict.items():
  455. tags.append(tag)
  456. if isPublic:
  457. updateHashtagsIndex(baseDir,tag,newPostId)
  458. if not clientToServer:
  459. actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
  460. # if capabilities have been granted for this actor
  461. # then get the corresponding id
  462. capabilityId=None
  463. capabilityIdList=[]
  464. ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
  465. if ocapFilename:
  466. if os.path.isfile(ocapFilename):
  467. with open(ocapFilename, 'r') as fp:
  468. oc=commentjson.load(fp)
  469. if oc.get('id'):
  470. capabilityIdList=[oc['id']]
  471. newPost = {
  472. "@context": "https://www.w3.org/ns/activitystreams",
  473. 'id': newPostId+'/activity',
  474. 'capability': capabilityIdList,
  475. 'type': 'Create',
  476. 'actor': actorUrl,
  477. 'published': published,
  478. 'to': [toUrl],
  479. 'cc': [],
  480. 'object': {
  481. 'id': newPostId,
  482. 'type': 'Note',
  483. 'summary': summary,
  484. 'inReplyTo': inReplyTo,
  485. 'published': published,
  486. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  487. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  488. 'to': toRecipients,
  489. 'cc': [],
  490. 'sensitive': sensitive,
  491. 'atomUri': newPostId,
  492. 'inReplyToAtomUri': inReplyToAtomUri,
  493. 'content': content,
  494. 'contentMap': {
  495. 'en': content
  496. },
  497. 'attachment': [],
  498. 'tag': tags,
  499. 'replies': {
  500. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  501. 'type': 'Collection',
  502. 'first': {
  503. 'type': 'CollectionPage',
  504. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  505. 'items': []
  506. }
  507. }
  508. }
  509. }
  510. if attachImageFilename:
  511. newPost['object']= \
  512. attachMedia(baseDir,httpPrefix,domain,port, \
  513. newPost['object'],attachImageFilename, \
  514. mediaType,imageDescription,useBlurhash)
  515. else:
  516. newPost = {
  517. "@context": "https://www.w3.org/ns/activitystreams",
  518. 'id': newPostId,
  519. 'type': 'Note',
  520. 'summary': summary,
  521. 'inReplyTo': inReplyTo,
  522. 'published': published,
  523. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  524. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  525. 'to': toRecipients,
  526. 'cc': [],
  527. 'sensitive': sensitive,
  528. 'atomUri': newPostId,
  529. 'inReplyToAtomUri': inReplyToAtomUri,
  530. 'content': content,
  531. 'contentMap': {
  532. 'en': content
  533. },
  534. 'attachment': [],
  535. 'tag': tags,
  536. 'replies': {
  537. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  538. 'type': 'Collection',
  539. 'first': {
  540. 'type': 'CollectionPage',
  541. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  542. 'items': []
  543. }
  544. }
  545. }
  546. if attachImageFilename:
  547. newPost= \
  548. attachMedia(baseDir,httpPrefix,domain,port, \
  549. newPost,attachImageFilename, \
  550. mediaType,imageDescription,useBlurhash)
  551. if ccUrl:
  552. if len(ccUrl)>0:
  553. newPost['cc']=[ccUrl]
  554. if newPost.get('object'):
  555. newPost['object']['cc']=[ccUrl]
  556. # if this is a moderation report then add a status
  557. if isModerationReport:
  558. # add status
  559. if newPost.get('object'):
  560. newPost['object']['moderationStatus']='pending'
  561. else:
  562. newPost['moderationStatus']='pending'
  563. # save to index file
  564. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  565. modFile=open(moderationIndexFile, "a+")
  566. if modFile:
  567. modFile.write(newPostId+'\n')
  568. modFile.close()
  569. if saveToFile:
  570. savePostToBox(baseDir,httpPrefix,newPostId, \
  571. nickname,domain,newPost,'outbox')
  572. return newPost
  573. def outboxMessageCreateWrap(httpPrefix: str, \
  574. nickname: str,domain: str,port: int, \
  575. messageJson: {}) -> {}:
  576. """Wraps a received message in a Create
  577. https://www.w3.org/TR/activitypub/#object-without-create
  578. """
  579. if port:
  580. if port!=80 and port!=443:
  581. if ':' not in domain:
  582. domain=domain+':'+str(port)
  583. statusNumber,published = getStatusNumber()
  584. if messageJson.get('published'):
  585. published = messageJson['published']
  586. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  587. cc=[]
  588. if messageJson.get('cc'):
  589. cc=messageJson['cc']
  590. # TODO
  591. capabilityUrl=[]
  592. newPost = {
  593. "@context": "https://www.w3.org/ns/activitystreams",
  594. 'id': newPostId+'/activity',
  595. 'capability': capabilityUrl,
  596. 'type': 'Create',
  597. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  598. 'published': published,
  599. 'to': messageJson['to'],
  600. 'cc': cc,
  601. 'object': messageJson
  602. }
  603. newPost['object']['id']=newPost['id']
  604. newPost['object']['url']= \
  605. httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
  606. newPost['object']['atomUri']= \
  607. httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  608. return newPost
  609. def postIsAddressedToFollowers(baseDir: str, \
  610. nickname: str,domain: str, \
  611. port: int,httpPrefix: str, \
  612. postJsonObject: {}) -> bool:
  613. """Returns true if the given post is addressed to followers of the nickname
  614. """
  615. if port:
  616. if port!=80 and port!=443:
  617. if ':' not in domain:
  618. domain=domain+':'+str(port)
  619. if not postJsonObject.get('object'):
  620. return False
  621. toList=[]
  622. ccList=[]
  623. if postJsonObject['type']!='Update' and \
  624. isinstance(postJsonObject['object'], dict):
  625. if not postJsonObject['object'].get('to'):
  626. return False
  627. toList=postJsonObject['object']['to']
  628. if postJsonObject['object'].get('cc'):
  629. ccList=postJsonObject['object']['cc']
  630. else:
  631. if not postJsonObject.get('to'):
  632. return False
  633. toList=postJsonObject['to']
  634. if postJsonObject.get('cc'):
  635. ccList=postJsonObject['cc']
  636. followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  637. # does the followers url exist in 'to' or 'cc' lists?
  638. addressedToFollowers=False
  639. if followersUrl in toList:
  640. addressedToFollowers=True
  641. if not addressedToFollowers:
  642. if followersUrl in ccList:
  643. addressedToFollowers=True
  644. return addressedToFollowers
  645. def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
  646. """Returns true if the given post is addressed to public
  647. """
  648. if not postJsonObject.get('object'):
  649. return False
  650. if not postJsonObject['object'].get('to'):
  651. return False
  652. publicUrl='https://www.w3.org/ns/activitystreams#Public'
  653. # does the public url exist in 'to' or 'cc' lists?
  654. addressedToPublic=False
  655. if publicUrl in postJsonObject['object']['to']:
  656. addressedToPublic=True
  657. if not addressedToPublic:
  658. if not postJsonObject['object'].get('cc'):
  659. return False
  660. if publicUrl in postJsonObject['object']['cc']:
  661. addressedToPublic=True
  662. return addressedToPublic
  663. def createPublicPost(baseDir: str,
  664. nickname: str, domain: str, port: int,httpPrefix: str, \
  665. content: str, followersOnly: bool, saveToFile: bool,
  666. clientToServer: bool,\
  667. attachImageFilename: str,mediaType: str, \
  668. imageDescription: str,useBlurhash: bool, \
  669. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  670. """Public post
  671. """
  672. domainFull=domain
  673. if port:
  674. if port!=80 and port!=443:
  675. if ':' not in domain:
  676. domainFull=domain+':'+str(port)
  677. return createPostBase(baseDir,nickname, domain, port, \
  678. 'https://www.w3.org/ns/activitystreams#Public', \
  679. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  680. httpPrefix, content, followersOnly, saveToFile, \
  681. clientToServer, \
  682. attachImageFilename,mediaType, \
  683. imageDescription,useBlurhash, \
  684. False,inReplyTo,inReplyToAtomUri,subject)
  685. def createUnlistedPost(baseDir: str,
  686. nickname: str, domain: str, port: int,httpPrefix: str, \
  687. content: str, followersOnly: bool, saveToFile: bool,
  688. clientToServer: bool,\
  689. attachImageFilename: str,mediaType: str, \
  690. imageDescription: str,useBlurhash: bool, \
  691. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  692. """Unlisted post. This has the #Public and followers links inverted.
  693. """
  694. domainFull=domain
  695. if port:
  696. if port!=80 and port!=443:
  697. if ':' not in domain:
  698. domainFull=domain+':'+str(port)
  699. return createPostBase(baseDir,nickname, domain, port, \
  700. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  701. 'https://www.w3.org/ns/activitystreams#Public', \
  702. httpPrefix, content, followersOnly, saveToFile, \
  703. clientToServer, \
  704. attachImageFilename,mediaType, \
  705. imageDescription,useBlurhash, \
  706. False,inReplyTo, inReplyToAtomUri, subject)
  707. def createFollowersOnlyPost(baseDir: str,
  708. nickname: str, domain: str, port: int,httpPrefix: str, \
  709. content: str, followersOnly: bool, saveToFile: bool,
  710. clientToServer: bool,\
  711. attachImageFilename: str,mediaType: str, \
  712. imageDescription: str,useBlurhash: bool, \
  713. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  714. """Followers only post
  715. """
  716. domainFull=domain
  717. if port:
  718. if port!=80 and port!=443:
  719. if ':' not in domain:
  720. domainFull=domain+':'+str(port)
  721. return createPostBase(baseDir,nickname, domain, port, \
  722. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  723. None,
  724. httpPrefix, content, followersOnly, saveToFile, \
  725. clientToServer, \
  726. attachImageFilename,mediaType, \
  727. imageDescription,useBlurhash, \
  728. False,inReplyTo, inReplyToAtomUri, subject)
  729. def getMentionedPeople(baseDir: str,httpPrefix: str, \
  730. content: str,domain: str,debug: bool) -> []:
  731. """Extracts a list of mentioned actors from the given message content
  732. """
  733. if '@' not in content:
  734. return None
  735. mentions=[]
  736. words=content.split(' ')
  737. for wrd in words:
  738. if wrd.startswith('@'):
  739. handle=wrd[1:]
  740. if debug:
  741. print('DEBUG: mentioned handle '+handle)
  742. if '@' not in handle:
  743. handle=handle+'@'+domain
  744. if not os.path.isdir(baseDir+'/accounts/'+handle):
  745. continue
  746. else:
  747. externalDomain=handle.split('@')[1]
  748. if not ('.' in externalDomain or externalDomain=='localhost'):
  749. continue
  750. mentionedNickname=handle.split('@')[0]
  751. mentionedDomain=handle.split('@')[1].strip('\n')
  752. if ':' in mentionedDomain:
  753. mentionedDomain=mentionedDomain.split(':')[0]
  754. if not validNickname(mentionedDomain,mentionedNickname):
  755. continue
  756. actor=httpPrefix+'://'+handle.split('@')[1]+'/users/'+mentionedNickname
  757. mentions.append(actor)
  758. return mentions
  759. def createDirectMessagePost(baseDir: str, \
  760. nickname: str,domain: str,port: int, \
  761. httpPrefix: str, \
  762. content: str,followersOnly: bool,saveToFile: bool,
  763. clientToServer: bool,\
  764. attachImageFilename: str,mediaType: str, \
  765. imageDescription: str,useBlurhash: bool, \
  766. inReplyTo=None,inReplyToAtomUri=None, \
  767. subject=None,debug=False) -> {}:
  768. """Direct Message post
  769. """
  770. mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
  771. if debug:
  772. print('mentionedPeople: '+str(mentionedPeople))
  773. if not mentionedPeople:
  774. return None
  775. postTo=None
  776. postCc=None
  777. return createPostBase(baseDir,nickname, domain, port, \
  778. postTo,postCc, \
  779. httpPrefix, content, followersOnly, saveToFile, \
  780. clientToServer, \
  781. attachImageFilename,mediaType, \
  782. imageDescription,useBlurhash, \
  783. False,inReplyTo, inReplyToAtomUri, subject)
  784. def createReportPost(baseDir: str, \
  785. nickname: str,domain: str,port: int,httpPrefix: str, \
  786. content: str, followersOnly: bool, saveToFile: bool, \
  787. clientToServer: bool,\
  788. attachImageFilename: str,mediaType: str, \
  789. imageDescription: str,useBlurhash: bool, \
  790. debug: bool,subject=None) -> {}:
  791. """Send a report to moderators
  792. """
  793. domainFull=domain
  794. if port:
  795. if port!=80 and port!=443:
  796. if ':' not in domain:
  797. domainFull=domain+':'+str(port)
  798. # add a title to distinguish moderation reports from other posts
  799. reportTitle='Moderation Report'
  800. if not subject:
  801. subject=reportTitle
  802. else:
  803. if not subject.startswith(reportTitle):
  804. subject=reportTitle+': '+subject
  805. # create the list of moderators from the moderators file
  806. moderatorsList=[]
  807. moderatorsFile=baseDir+'/accounts/moderators.txt'
  808. if os.path.isfile(moderatorsFile):
  809. with open (moderatorsFile, "r") as fileHandler:
  810. for line in fileHandler:
  811. line=line.strip('\n')
  812. if line.startswith('#'):
  813. continue
  814. if line.startswith('/users/'):
  815. line=line.replace('users','')
  816. if line.startswith('@'):
  817. line=line[1:]
  818. if '@' in line:
  819. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
  820. if moderatorActor not in moderatorList:
  821. moderatorsList.append(moderatorActor)
  822. continue
  823. if line.startswith('http') or line.startswith('dat'):
  824. # must be a local address - no remote moderators
  825. if '://'+domainFull+'/' in line:
  826. if line not in moderatorsList:
  827. moderatorsList.append(line)
  828. else:
  829. if '/' not in line:
  830. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
  831. if moderatorActor not in moderatorsList:
  832. moderatorsList.append(moderatorActor)
  833. if len(moderatorsList)==0:
  834. # if there are no moderators then the admin becomes the moderator
  835. adminNickname=getConfigParam(baseDir,'admin')
  836. if adminNickname:
  837. moderatorsList.append(httpPrefix+'://'+domainFull+'/users/'+adminNickname)
  838. if not moderatorsList:
  839. return None
  840. if debug:
  841. print('DEBUG: Sending report to moderators')
  842. print(str(moderatorsList))
  843. postTo=moderatorsList
  844. postCc=None
  845. postJsonObject=None
  846. for toUrl in postTo:
  847. postJsonObject= \
  848. createPostBase(baseDir,nickname, domain, port, \
  849. toUrl,postCc, \
  850. httpPrefix, content, followersOnly, saveToFile, \
  851. clientToServer, \
  852. attachImageFilename,mediaType, \
  853. imageDescription,useBlurhash, \
  854. True,None, None, subject)
  855. return postJsonObject
  856. def threadSendPost(session,postJsonStr: str,federationList: [], \
  857. inboxUrl: str, baseDir: str, \
  858. signatureHeaderJson: {},postLog: [], \
  859. debug :bool) -> None:
  860. """Sends a post with exponential backoff
  861. """
  862. tries=0
  863. backoffTime=60
  864. for attempt in range(20):
  865. postResult = \
  866. postJsonString(session,postJsonStr,federationList, \
  867. inboxUrl,signatureHeaderJson, \
  868. "inbox:write",debug)
  869. if postResult:
  870. logStr='Success on try '+str(tries)+': '+postJsonStr
  871. else:
  872. logStr='Retry '+str(tries)+': '+postJsonStr
  873. postLog.append(logStr)
  874. # keep the length of the log finite
  875. # Don't accumulate massive files on systems with limited resources
  876. while len(postLog)>16:
  877. postLog.pop(0)
  878. # save the log file
  879. postLogFilename=baseDir+'/post.log'
  880. with open(postLogFilename, "a+") as logFile:
  881. logFile.write(logStr+'\n')
  882. if postResult:
  883. if debug:
  884. print('DEBUG: successful json post to '+inboxUrl)
  885. # our work here is done
  886. break
  887. if debug:
  888. print(postJsonStr)
  889. print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
  890. str(backoffTime)+' seconds.')
  891. time.sleep(backoffTime)
  892. backoffTime *= 2
  893. tries+=1
  894. def sendPost(projectVersion: str, \
  895. session,baseDir: str,nickname: str,domain: str,port: int, \
  896. toNickname: str,toDomain: str,toPort: int,cc: str, \
  897. httpPrefix: str,content: str,followersOnly: bool, \
  898. saveToFile: bool,clientToServer: bool, \
  899. attachImageFilename: str,mediaType: str, \
  900. imageDescription: str,useBlurhash: bool, \
  901. federationList: [], \
  902. sendThreads: [],postLog: [], \
  903. cachedWebfingers: {},personCache: {}, \
  904. debug=False,inReplyTo=None, \
  905. inReplyToAtomUri=None,subject=None) -> int:
  906. """Post to another inbox
  907. """
  908. withDigest=True
  909. if toNickname=='inbox':
  910. # shared inbox actor on @domain@domain
  911. toNickname=toDomain
  912. toDomainOriginal=toDomain
  913. if toPort:
  914. if toPort!=80 and toPort!=443:
  915. if ':' not in toDomain:
  916. toDomain=toDomain+':'+str(toPort)
  917. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  918. # lookup the inbox for the To handle
  919. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  920. domain,projectVersion)
  921. if not wfRequest:
  922. return 1
  923. if not clientToServer:
  924. postToBox='inbox'
  925. else:
  926. postToBox='outbox'
  927. # get the actor inbox for the To handle
  928. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  929. getPersonBox(baseDir,session,wfRequest,personCache, \
  930. projectVersion,httpPrefix,domain,postToBox)
  931. # If there are more than one followers on the target domain
  932. # then send to the shared inbox indead of the individual inbox
  933. if nickname=='capabilities':
  934. inboxUrl=capabilityAcquisition
  935. if not capabilityAcquisition:
  936. return 2
  937. if not inboxUrl:
  938. return 3
  939. if not pubKey:
  940. return 4
  941. if not toPersonId:
  942. return 5
  943. # sharedInbox and capabilities are optional
  944. postJsonObject = \
  945. createPostBase(baseDir,nickname,domain,port, \
  946. toPersonId,cc,httpPrefix,content, \
  947. followersOnly,saveToFile,clientToServer, \
  948. attachImageFilename,mediaType, \
  949. imageDescription,useBlurhash, \
  950. False,inReplyTo,inReplyToAtomUri,subject)
  951. # get the senders private key
  952. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
  953. if len(privateKeyPem)==0:
  954. return 6
  955. if toDomain not in inboxUrl:
  956. return 7
  957. postPath=inboxUrl.split(toDomain,1)[1]
  958. # convert json to string so that there are no
  959. # subsequent conversions after creating message body digest
  960. postJsonStr=json.dumps(postJsonObject)
  961. # construct the http header, including the message body digest
  962. signatureHeaderJson = \
  963. createSignedHeader(privateKeyPem,nickname,domain,port, \
  964. toDomain,toPort, \
  965. postPath,httpPrefix,withDigest,postJsonStr)
  966. # Keep the number of threads being used small
  967. while len(sendThreads)>10:
  968. sendThreads[0].kill()
  969. sendThreads.pop(0)
  970. thr = threadWithTrace(target=threadSendPost,args=(session, \
  971. postJsonStr, \
  972. federationList, \
  973. inboxUrl,baseDir, \
  974. signatureHeaderJson.copy(), \
  975. postLog,
  976. debug),daemon=True)
  977. sendThreads.append(thr)
  978. thr.start()
  979. return 0
  980. def sendPostViaServer(projectVersion: str, \
  981. baseDir: str,session,fromNickname: str,password: str, \
  982. fromDomain: str, fromPort: int, \
  983. toNickname: str, toDomain: str, toPort: int, cc: str, \
  984. httpPrefix: str, content: str, followersOnly: bool, \
  985. attachImageFilename: str,mediaType: str, \
  986. imageDescription: str,useBlurhash: bool, \
  987. cachedWebfingers: {},personCache: {}, \
  988. debug=False,inReplyTo=None, \
  989. inReplyToAtomUri=None,subject=None) -> int:
  990. """Send a post via a proxy (c2s)
  991. """
  992. if not session:
  993. print('WARN: No session for sendPostViaServer')
  994. return 6
  995. withDigest=True
  996. if toPort:
  997. if toPort!=80 and toPort!=443:
  998. if ':' not in fromDomain:
  999. fromDomain=fromDomain+':'+str(fromPort)
  1000. handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
  1001. # lookup the inbox for the To handle
  1002. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1003. fromDomain,projectVersion)
  1004. if not wfRequest:
  1005. if debug:
  1006. print('DEBUG: webfinger failed for '+handle)
  1007. return 1
  1008. postToBox='outbox'
  1009. # get the actor inbox for the To handle
  1010. inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1011. getPersonBox(baseDir,session,wfRequest,personCache, \
  1012. projectVersion,httpPrefix,fromDomain,postToBox)
  1013. if not inboxUrl:
  1014. if debug:
  1015. print('DEBUG: No '+postToBox+' was found for '+handle)
  1016. return 3
  1017. if not fromPersonId:
  1018. if debug:
  1019. print('DEBUG: No actor was found for '+handle)
  1020. return 4
  1021. # Get the json for the c2s post, not saving anything to file
  1022. # Note that baseDir is set to None
  1023. saveToFile=False
  1024. clientToServer=True
  1025. if toDomain.lower().endswith('public'):
  1026. toPersonId='https://www.w3.org/ns/activitystreams#Public'
  1027. fromDomainFull=fromDomain
  1028. if fromPort:
  1029. if fromPort!=80 and fromPort!=443:
  1030. if ':' not in fromDomain:
  1031. fromDomainFull=fromDomain+':'+str(fromPort)
  1032. cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1033. else:
  1034. if toDomain.lower().endswith('followers') or \
  1035. toDomain.lower().endswith('followersonly'):
  1036. toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1037. else:
  1038. toDomainFull=toDomain
  1039. if toPort:
  1040. if toPort!=80 and toPort!=443:
  1041. if ':' not in toDomain:
  1042. toDomainFull=toDomain+':'+str(toPort)
  1043. toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
  1044. postJsonObject = \
  1045. createPostBase(baseDir, \
  1046. fromNickname,fromDomain,fromPort, \
  1047. toPersonId,cc,httpPrefix,content, \
  1048. followersOnly,saveToFile,clientToServer, \
  1049. attachImageFilename,mediaType, \
  1050. imageDescription,useBlurhash, \
  1051. False,inReplyTo,inReplyToAtomUri,subject)
  1052. authHeader=createBasicAuthHeader(fromNickname,password)
  1053. if attachImageFilename:
  1054. headers = {'host': fromDomain, \
  1055. 'Authorization': authHeader}
  1056. postResult = \
  1057. postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
  1058. #if not postResult:
  1059. # if debug:
  1060. # print('DEBUG: Failed to upload image')
  1061. # return 9
  1062. headers = {'host': fromDomain, \
  1063. 'Content-type': 'application/json', \
  1064. 'Authorization': authHeader}
  1065. postResult = \
  1066. postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write",debug)
  1067. #if not postResult:
  1068. # if debug:
  1069. # print('DEBUG: POST failed for c2s to '+inboxUrl)
  1070. # return 5
  1071. if debug:
  1072. print('DEBUG: c2s POST success')
  1073. return 0
  1074. def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
  1075. """Returns a dictionary with followers grouped by domain
  1076. """
  1077. handle=nickname+'@'+domain
  1078. followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
  1079. if not os.path.isfile(followersFilename):
  1080. return None
  1081. grouped={}
  1082. with open(followersFilename, "r") as f:
  1083. for followerHandle in f:
  1084. if '@' in followerHandle:
  1085. fHandle=followerHandle.strip().replace('\n','')
  1086. followerDomain=fHandle.split('@')[1]
  1087. if not grouped.get(followerDomain):
  1088. grouped[followerDomain]=[fHandle]
  1089. else:
  1090. grouped[followerDomain].append(fHandle)
  1091. return grouped
  1092. def sendSignedJson(postJsonObject: {},session,baseDir: str, \
  1093. nickname: str, domain: str, port: int, \
  1094. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1095. httpPrefix: str, saveToFile: bool, clientToServer: bool, \
  1096. federationList: [], \
  1097. sendThreads: [], postLog: [], cachedWebfingers: {}, \
  1098. personCache: {}, debug: bool,projectVersion: str) -> int:
  1099. """Sends a signed json object to an inbox/outbox
  1100. """
  1101. if debug:
  1102. print('DEBUG: sendSignedJson start')
  1103. if not session:
  1104. print('WARN: No session specified for sendSignedJson')
  1105. return 8
  1106. withDigest=True
  1107. sharedInbox=False
  1108. if toNickname=='inbox':
  1109. # shared inbox actor on @domain@domain
  1110. toNickname=toDomain
  1111. sharedInbox=True
  1112. toDomainOriginal=toDomain
  1113. if toPort:
  1114. if toPort!=80 and toPort!=443:
  1115. if ':' not in toDomain:
  1116. toDomain=toDomain+':'+str(toPort)
  1117. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  1118. if debug:
  1119. print('DEBUG: handle - '+handle+' toPort '+str(toPort))
  1120. # lookup the inbox for the To handle
  1121. wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1122. domain,projectVersion)
  1123. if not wfRequest:
  1124. if debug:
  1125. print('DEBUG: webfinger for '+handle+' failed')
  1126. return 1
  1127. if not clientToServer:
  1128. postToBox='inbox'
  1129. else:
  1130. postToBox='outbox'
  1131. # get the actor inbox/outbox/capabilities for the To handle
  1132. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
  1133. getPersonBox(baseDir,session,wfRequest,personCache, \
  1134. projectVersion,httpPrefix,domain,postToBox)
  1135. if nickname=='capabilities':
  1136. inboxUrl=capabilityAcquisition
  1137. if not capabilityAcquisition:
  1138. return 2
  1139. else:
  1140. print("inboxUrl: "+inboxUrl)
  1141. print("toPersonId: "+toPersonId)
  1142. print("sharedInboxUrl: "+sharedInboxUrl)
  1143. if inboxUrl.endswith('/actor/inbox'):
  1144. inboxUrl=sharedInboxUrl
  1145. if not inboxUrl:
  1146. if debug:
  1147. print('DEBUG: missing inboxUrl')
  1148. return 3
  1149. if debug:
  1150. print('DEBUG: Sending to endpoint '+inboxUrl)
  1151. if not pubKey:
  1152. if debug:
  1153. print('DEBUG: missing pubkey')
  1154. return 4
  1155. if not toPersonId:
  1156. if debug:
  1157. print('DEBUG: missing personId')
  1158. return 5
  1159. # sharedInbox and capabilities are optional
  1160. # get the senders private key
  1161. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
  1162. if len(privateKeyPem)==0:
  1163. if debug:
  1164. print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
  1165. return 6
  1166. if toDomain not in inboxUrl:
  1167. if debug:
  1168. print('DEBUG: '+toDomain+' is not in '+inboxUrl)
  1169. return 7
  1170. postPath=inboxUrl.split(toDomain,1)[1]
  1171. # convert json to string so that there are no
  1172. # subsequent conversions after creating message body digest
  1173. postJsonStr=json.dumps(postJsonObject)
  1174. # construct the http header, including the message body digest
  1175. signatureHeaderJson = \
  1176. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1177. toDomain,toPort, \
  1178. postPath,httpPrefix,withDigest,postJsonStr)
  1179. # Keep the number of threads being used small
  1180. while len(sendThreads)>10:
  1181. sendThreads[0].kill()
  1182. sendThreads.pop(0)
  1183. if debug:
  1184. print('DEBUG: starting thread to send post')
  1185. pprint(postJsonObject)
  1186. thr = threadWithTrace(target=threadSendPost, \
  1187. args=(session, \
  1188. postJsonStr, \
  1189. federationList, \
  1190. inboxUrl,baseDir, \
  1191. signatureHeaderJson.copy(), \
  1192. postLog,
  1193. debug),daemon=True)
  1194. sendThreads.append(thr)
  1195. thr.start()
  1196. return 0
  1197. def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
  1198. """The Follow activity doesn't have a 'to' field and so one
  1199. needs to be added so that activity distribution happens in a consistent way
  1200. Returns true if a 'to' field exists or was added
  1201. """
  1202. if postJsonObject.get('to'):
  1203. return postJsonObject,True
  1204. if debug:
  1205. pprint(postJsonObject)
  1206. print('DEBUG: no "to" field when sending to named addresses 2')
  1207. isSameType=False
  1208. toFieldAdded=False
  1209. if postJsonObject.get('object'):
  1210. if isinstance(postJsonObject['object'], str):
  1211. if postJsonObject.get('type'):
  1212. if postJsonObject['type']==activityType:
  1213. isSameType=True
  1214. if debug:
  1215. print('DEBUG: "to" field assigned to Follow')
  1216. toAddress=postJsonObject['object']
  1217. if '/statuses/' in toAddress:
  1218. toAddress=toAddress.split('/statuses/')[0]
  1219. postJsonObject['to']=[toAddress]
  1220. toFieldAdded=True
  1221. elif isinstance(postJsonObject['object'], dict):
  1222. if postJsonObject['object'].get('type'):
  1223. if postJsonObject['object']['type']==activityType:
  1224. isSameType=True
  1225. if isinstance(postJsonObject['object']['object'], str):
  1226. if debug:
  1227. print('DEBUG: "to" field assigned to Follow')
  1228. toAddress=postJsonObject['object']['object']
  1229. if '/statuses/' in toAddress:
  1230. toAddress=toAddress.split('/statuses/')[0]
  1231. postJsonObject['object']['to']=[toAddress]
  1232. postJsonObject['to']=[postJsonObject['object']['object']]
  1233. toFieldAdded=True
  1234. if not isSameType:
  1235. return postJsonObject,True
  1236. if toFieldAdded:
  1237. return postJsonObject,True
  1238. return postJsonObject,False
  1239. def sendToNamedAddresses(session,baseDir: str, \
  1240. nickname: str, domain: str, port: int, \
  1241. httpPrefix: str,federationList: [], \
  1242. sendThreads: [],postLog: [], \
  1243. cachedWebfingers: {},personCache: {}, \
  1244. postJsonObject: {},debug: bool, \
  1245. projectVersion: str) -> None:
  1246. """sends a post to the specific named addresses in to/cc
  1247. """
  1248. if not session:
  1249. print('WARN: No session for sendToNamedAddresses')
  1250. return
  1251. if not postJsonObject.get('object'):
  1252. return
  1253. if isinstance(postJsonObject['object'], dict):
  1254. isProfileUpdate=False
  1255. # for actor updates there is no 'to' within the object
  1256. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1257. if postJsonObject['type']=='Update' and \
  1258. (postJsonObject['object']['type']=='Person' or \
  1259. postJsonObject['object']['type']=='Application' or \
  1260. postJsonObject['object']['type']=='Service'):
  1261. # use the original object, which has a 'to'
  1262. recipientsObject=postJsonObject
  1263. isProfileUpdate=True
  1264. if not isProfileUpdate:
  1265. if not postJsonObject['object'].get('to'):
  1266. if debug:
  1267. pprint(postJsonObject)
  1268. print('DEBUG: no "to" field when sending to named addresses')
  1269. if postJsonObject['object'].get('type'):
  1270. if postJsonObject['object']['type']=='Follow':
  1271. if isinstance(postJsonObject['object']['object'], str):
  1272. if debug:
  1273. print('DEBUG: "to" field assigned to Follow')
  1274. postJsonObject['object']['to']=[postJsonObject['object']['object']]
  1275. if not postJsonObject['object'].get('to'):
  1276. return
  1277. recipientsObject=postJsonObject['object']
  1278. else:
  1279. postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
  1280. if not fieldAdded:
  1281. return
  1282. postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
  1283. if not fieldAdded:
  1284. return
  1285. recipientsObject=postJsonObject
  1286. recipients=[]
  1287. recipientType=['to','cc']
  1288. for rType in recipientType:
  1289. if not recipientsObject.get(rType):
  1290. continue
  1291. if isinstance(recipientsObject[rType], list):
  1292. if debug:
  1293. pprint(recipientsObject)
  1294. print('recipientsObject: '+str(recipientsObject[rType]))
  1295. for address in recipientsObject[rType]:
  1296. if not address:
  1297. continue
  1298. if '/' not in address:
  1299. continue
  1300. if address.endswith('#Public'):
  1301. continue
  1302. if address.endswith('/followers'):
  1303. continue
  1304. recipients.append(address)
  1305. elif isinstance(recipientsObject[rType], str):
  1306. address=recipientsObject[rType]
  1307. if address:
  1308. if '/' in address:
  1309. if address.endswith('#Public'):
  1310. continue
  1311. if address.endswith('/followers'):
  1312. continue
  1313. recipients.append(address)
  1314. if not recipients:
  1315. if debug:
  1316. print('DEBUG: no individual recipients')
  1317. return
  1318. if debug:
  1319. print('DEBUG: Sending individually addressed posts: '+str(recipients))
  1320. # this is after the message has arrived at the server
  1321. clientToServer=False
  1322. for address in recipients:
  1323. toNickname=getNicknameFromActor(address)
  1324. if not toNickname:
  1325. continue
  1326. toDomain,toPort=getDomainFromActor(address)
  1327. if not toDomain:
  1328. continue
  1329. if debug:
  1330. domainFull=domain
  1331. if port:
  1332. if port!=80 and port!=443:
  1333. if ':' not in domain:
  1334. domainFull=domain+':'+str(port)
  1335. toDomainFull=toDomain
  1336. if toPort:
  1337. if toPort!=80 and toPort!=443:
  1338. if ':' not in toDomain:
  1339. toDomainFull=toDomain+':'+str(toPort)
  1340. print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
  1341. cc=[]
  1342. sendSignedJson(postJsonObject,session,baseDir, \
  1343. nickname,domain,port, \
  1344. toNickname,toDomain,toPort, \
  1345. cc,httpPrefix,True,clientToServer, \
  1346. federationList, \
  1347. sendThreads,postLog,cachedWebfingers, \
  1348. personCache,debug,projectVersion)
  1349. def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
  1350. """Returns true if the given domain has a shared inbox
  1351. """
  1352. wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
  1353. None,__version__)
  1354. if wfRequest:
  1355. return True
  1356. return False
  1357. def sendToFollowers(session,baseDir: str, \
  1358. nickname: str, domain: str, port: int, \
  1359. httpPrefix: str,federationList: [], \
  1360. sendThreads: [],postLog: [], \
  1361. cachedWebfingers: {},personCache: {}, \
  1362. postJsonObject: {},debug: bool, \
  1363. projectVersion: str) -> None:
  1364. """sends a post to the followers of the given nickname
  1365. """
  1366. print('sendToFollowers')
  1367. if not session:
  1368. print('WARN: No session for sendToFollowers')
  1369. return
  1370. if not postIsAddressedToFollowers(baseDir,nickname,domain, \
  1371. port,httpPrefix,postJsonObject):
  1372. if debug:
  1373. print('Post is not addressed to followers')
  1374. return
  1375. print('Post is addressed to followers')
  1376. grouped=groupFollowersByDomain(baseDir,nickname,domain)
  1377. if not grouped:
  1378. if debug:
  1379. print('Post to followers did not resolve any domains')
  1380. return
  1381. print('Post to followers resolved domains')
  1382. print(str(grouped))
  1383. # this is after the message has arrived at the server
  1384. clientToServer=False
  1385. # for each instance
  1386. for followerDomain,followerHandles in grouped.items():
  1387. if debug:
  1388. print('DEBUG: follower handles for '+followerDomain)
  1389. pprint(followerHandles)
  1390. withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
  1391. if debug:
  1392. if withSharedInbox:
  1393. print(followerDomain+' has shared inbox')
  1394. else:
  1395. print(followerDomain+' does not have a shared inbox')
  1396. toPort=port
  1397. index=0
  1398. toDomain=followerHandles[index].split('@')[1]
  1399. if ':' in toDomain:
  1400. toPort=toDomain.split(':')[1]
  1401. toDomain=toDomain.split(':')[0]
  1402. # If this is a profile update then send to shared inbox
  1403. cc=''
  1404. if withSharedInbox:
  1405. toNickname=followerHandles[index].split('@')[0]
  1406. # if there are more than one followers on the domain
  1407. # then send the post to the shared inbox
  1408. if len(followerHandles)>1:
  1409. toNickname='inbox'
  1410. if toNickname!='inbox' and postJsonObject.get('type'):
  1411. if postJsonObject['type']=='Update':
  1412. if postJsonObject.get('object'):
  1413. if isinstance(postJsonObject['object'], dict):
  1414. if postJsonObject['object'].get('type'):
  1415. if postJsonObject['object']['type']=='Person' or \
  1416. postJsonObject['object']['type']=='Application' or \
  1417. postJsonObject['object']['type']=='Service':
  1418. print('Sending profile update to shared inbox of '+toDomain)
  1419. toNickname='inbox'
  1420. if debug:
  1421. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1422. sendSignedJson(postJsonObject,session,baseDir, \
  1423. nickname,domain,port, \
  1424. toNickname,toDomain,toPort, \
  1425. cc,httpPrefix,True,clientToServer, \
  1426. federationList, \
  1427. sendThreads,postLog,cachedWebfingers, \
  1428. personCache,debug,projectVersion)
  1429. else:
  1430. # send to individual followers without using a shared inbox
  1431. for handle in followerHandles:
  1432. toNickname=handle.split('@')[0]
  1433. if debug:
  1434. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1435. sendSignedJson(postJsonObject,session,baseDir, \
  1436. nickname,domain,port, \
  1437. toNickname,toDomain,toPort, \
  1438. cc,httpPrefix,True,clientToServer, \
  1439. federationList, \
  1440. sendThreads,postLog,cachedWebfingers, \
  1441. personCache,debug,projectVersion)
  1442. if debug:
  1443. print('DEBUG: End of sendToFollowers')
  1444. def createInbox(baseDir: str,nickname: str,domain: str,port: int, \
  1445. httpPrefix: str, \
  1446. itemsPerPage: int,headerOnly: bool, \
  1447. ocapAlways: bool,pageNumber=None) -> {}:
  1448. return createBoxBase(baseDir,'inbox',nickname,domain,port,httpPrefix, \
  1449. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1450. def createDMTimeline(baseDir: str,nickname: str,domain: str,port: int, \
  1451. httpPrefix: str, \
  1452. itemsPerPage: int,headerOnly: bool, \
  1453. ocapAlways: bool,pageNumber=None) -> {}:
  1454. return createBoxBase(baseDir,'dm',nickname,domain,port,httpPrefix, \
  1455. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1456. def createOutbox(baseDir: str,nickname: str,domain: str,port: int, \
  1457. httpPrefix: str, \
  1458. itemsPerPage: int,headerOnly: bool, \
  1459. authorized: bool,pageNumber=None) -> {}:
  1460. return createBoxBase(baseDir,'outbox',nickname,domain,port,httpPrefix, \
  1461. itemsPerPage,headerOnly,authorized,False,pageNumber)
  1462. def createModeration(baseDir: str,nickname: str,domain: str,port: int, \
  1463. httpPrefix: str, \
  1464. itemsPerPage: int,headerOnly: bool, \
  1465. ocapAlways: bool,pageNumber=None) -> {}:
  1466. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1467. boxname='moderation'
  1468. if port:
  1469. if port!=80 and port!=443:
  1470. if ':' not in domain:
  1471. domain=domain+':'+str(port)
  1472. if not pageNumber:
  1473. pageNumber=1
  1474. pageStr='?page='+str(pageNumber)
  1475. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1476. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1477. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1478. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1479. 'totalItems': 0,
  1480. 'type': 'OrderedCollection'}
  1481. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1482. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1483. 'orderedItems': [
  1484. ],
  1485. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1486. 'type': 'OrderedCollectionPage'}
  1487. if isModerator(baseDir,nickname):
  1488. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  1489. if os.path.isfile(moderationIndexFile):
  1490. with open(moderationIndexFile, "r") as f:
  1491. lines = f.readlines()
  1492. boxHeader['totalItems']=len(lines)
  1493. if headerOnly:
  1494. return boxHeader
  1495. pageLines=[]
  1496. if len(lines)>0:
  1497. endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
  1498. if endLineNumber<0:
  1499. endLineNumber=0
  1500. startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
  1501. if startLineNumber<0:
  1502. startLineNumber=0
  1503. lineNumber=startLineNumber
  1504. while lineNumber>=endLineNumber:
  1505. pageLines.append(lines[lineNumber].strip('\n'))
  1506. lineNumber-=1
  1507. for postUrl in pageLines:
  1508. postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
  1509. if os.path.isfile(postFilename):
  1510. with open(postFilename, 'r') as fp:
  1511. postJsonObject=commentjson.load(fp)
  1512. boxItems['orderedItems'].append(postJsonObject)
  1513. if headerOnly:
  1514. return boxHeader
  1515. return boxItems
  1516. def getStatusNumberFromPostFilename(filename) -> int:
  1517. """Gets the status number from a post filename
  1518. eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
  1519. returns 156295850695206
  1520. """
  1521. if '#statuses#' not in filename:
  1522. return None
  1523. return int(filename.split('#')[-1].replace('.json',''))
  1524. def isDM(postJsonObject: {}) -> bool:
  1525. """Returns true if the given post is a DM
  1526. """
  1527. if postJsonObject['type']!='Create':
  1528. return False
  1529. if not postJsonObject.get('object'):
  1530. return False
  1531. if not isinstance(postJsonObject['object'], dict):
  1532. return False
  1533. if postJsonObject['object']['type']!='Note':
  1534. return False
  1535. fields=['to','cc']
  1536. for f in fields:
  1537. if not postJsonObject['object'].get(f):
  1538. continue
  1539. for toAddress in postJsonObject['object'][f]:
  1540. if toAddress.endswith('#Public'):
  1541. return False
  1542. if toAddress.endswith('followers'):
  1543. return False
  1544. return True
  1545. def createBoxBase(baseDir: str,boxname: str, \
  1546. nickname: str,domain: str,port: int,httpPrefix: str, \
  1547. itemsPerPage: int,headerOnly: bool,authorized :bool, \
  1548. ocapAlways: bool,pageNumber=None) -> {}:
  1549. """Constructs the box feed for a person with the given nickname
  1550. """
  1551. if boxname!='inbox' and boxname!='dm' and boxname!='outbox':
  1552. return None
  1553. if boxname!='dm':
  1554. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1555. else:
  1556. # extract DMs from the inbox
  1557. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1558. sharedBoxDir=None
  1559. if boxname=='inbox':
  1560. sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
  1561. if port:
  1562. if port!=80 and port!=443:
  1563. if ':' not in domain:
  1564. domain=domain+':'+str(port)
  1565. pageStr='?page=true'
  1566. if pageNumber:
  1567. try:
  1568. pageStr='?page='+str(pageNumber)
  1569. except:
  1570. pass
  1571. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1572. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1573. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1574. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1575. 'totalItems': 0,
  1576. 'type': 'OrderedCollection'}
  1577. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1578. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1579. 'orderedItems': [
  1580. ],
  1581. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1582. 'type': 'OrderedCollectionPage'}
  1583. # counter for posts so far added to the target page
  1584. postsOnPageCtr=0
  1585. # post filenames sorted in descending order
  1586. postsInBoxDict={}
  1587. postsCtr=0
  1588. postsInPersonInbox=os.listdir(boxDir)
  1589. for postFilename in postsInPersonInbox:
  1590. if not postFilename.endswith('.json'):
  1591. continue
  1592. # extract the status number
  1593. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1594. if statusNumber:
  1595. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1596. postsCtr+=1
  1597. # combine the inbox for the account with the shared inbox
  1598. if sharedBoxDir:
  1599. handle=nickname+'@'+domain
  1600. followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
  1601. postsInSharedInbox=os.listdir(sharedBoxDir)
  1602. for postFilename in postsInSharedInbox:
  1603. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1604. if statusNumber:
  1605. sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
  1606. # get the actor from the shared post
  1607. with open(sharedInboxFilename, 'r') as fp:
  1608. postJsonObject=commentjson.load(fp)
  1609. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  1610. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  1611. if actorNickname and actorDomain:
  1612. # is the actor followed by this account?
  1613. if actorNickname+'@'+actorDomain in open(followingFilename).read():
  1614. if ocapAlways:
  1615. capsList=None
  1616. # Note: should this be in the Create or the object of a post?
  1617. if postJsonObject.get('capability'):
  1618. if isinstance(postJsonObject['capability'], list):
  1619. capsList=postJsonObject['capability']
  1620. # Have capabilities been granted for the sender?
  1621. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
  1622. if os.path.isfile(ocapFilename):
  1623. # read the capabilities id
  1624. with open(ocapFilename, 'r') as fp:
  1625. ocapJson=commentjson.load(fp)
  1626. if ocapJson.get('id'):
  1627. if ocapJson['id'] in capsList:
  1628. postsInBoxDict[statusNumber]=sharedInboxFilename
  1629. postsCtr+=1
  1630. else:
  1631. postsInBoxDict[statusNumber]=sharedInboxFilename
  1632. postsCtr+=1
  1633. # sort the list in descending order of date
  1634. postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=True))
  1635. # number of posts in box
  1636. boxHeader['totalItems']=postsCtr
  1637. prevPostFilename=None
  1638. if not pageNumber:
  1639. pageNumber=1
  1640. # Generate first and last entries within header
  1641. if postsCtr>0:
  1642. lastPage=int(postsCtr/itemsPerPage)
  1643. if lastPage<1:
  1644. lastPage=1
  1645. boxHeader['last']= \
  1646. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
  1647. # Insert posts
  1648. currPage=1
  1649. postsCtr=0
  1650. for statusNumber,postFilename in postsInBox.items():
  1651. # Are we at the starting page yet?
  1652. if prevPostFilename and currPage==pageNumber and postsCtr==0:
  1653. # update the prev entry for the last message id
  1654. postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
  1655. boxHeader['prev']= \
  1656. httpPrefix+'://'+domain+'/users/'+nickname+'/'+ \
  1657. boxname+'?min_id='+postId+'&page=true'
  1658. # get the full path of the post file
  1659. filePath = postFilename
  1660. try:
  1661. if os.path.isfile(filePath):
  1662. # is this a valid timeline post?
  1663. isPost=False
  1664. # must be a "Note" or "Announce" type
  1665. with open(filePath, 'r') as file:
  1666. postStr = file.read()
  1667. if '"Note"' in postStr or '"Announce"' in postStr or \
  1668. ('"Question"' in postStr and '"Create"' in postStr):
  1669. isPost=True
  1670. if boxname=='dm':
  1671. if '#Public' in postStr or '/followers' in postStr:
  1672. isPost=False
  1673. if isPost:
  1674. if currPage == pageNumber and postsOnPageCtr <= itemsPerPage:
  1675. # get the post as json
  1676. p = json.loads(postStr)
  1677. if boxname!='dm' or \
  1678. (boxname=='dm' and isDM(p)):
  1679. # remove any capability so that it's not displayed
  1680. if p.get('capability'):
  1681. del p['capability']
  1682. # Don't show likes or replies to unauthorized viewers
  1683. if not authorized:
  1684. if p.get('object'):
  1685. if isinstance(p['object'], dict):
  1686. if p['object'].get('likes'):
  1687. p['likes']={}
  1688. if p['object'].get('replies'):
  1689. p['replies']={}
  1690. # insert it into the box feed
  1691. if postsOnPageCtr < itemsPerPage:
  1692. if not headerOnly:
  1693. boxItems['orderedItems'].append(p)
  1694. postsOnPageCtr += 1
  1695. elif postsOnPageCtr == itemsPerPage:
  1696. # if this is the last post update the next message ID
  1697. if '/statuses/' in p['id']:
  1698. postId = p['id'].split('/statuses/')[1].replace('/activity','')
  1699. boxHeader['next']= \
  1700. httpPrefix+'://'+domain+'/users/'+ \
  1701. nickname+'/'+boxname+'?max_id='+ \
  1702. postId+'&page=true'
  1703. # remember the last post filename for use with prev
  1704. prevPostFilename = postFilename
  1705. if postsOnPageCtr >= itemsPerPage:
  1706. break
  1707. # count the pages
  1708. postsCtr += 1
  1709. if postsCtr >= itemsPerPage:
  1710. postsCtr = 0
  1711. currPage += 1
  1712. except Exception as e:
  1713. print(e)
  1714. if headerOnly:
  1715. return boxHeader
  1716. return boxItems
  1717. def expireCache(baseDir: str,personCache: {}, \
  1718. httpPrefix: str,archiveDir: str,maxPostsInBox=256):
  1719. """Thread used to expire actors from the cache and archive old posts
  1720. """
  1721. while True:
  1722. # once per day
  1723. time.sleep(60*60*24)
  1724. expirePersonCache(basedir,personCache)
  1725. archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
  1726. def archivePosts(baseDir: str,httpPrefix: str, \
  1727. archiveDir: str,maxPostsInBox=256) -> None:
  1728. """Archives posts for all accounts
  1729. """
  1730. if archiveDir:
  1731. if not os.path.isdir(archiveDir):
  1732. os.mkdir(archiveDir)
  1733. if archiveDir:
  1734. if not os.path.isdir(archiveDir+'/accounts'):
  1735. os.mkdir(archiveDir+'/accounts')
  1736. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  1737. for handle in dirs:
  1738. if '@' in handle:
  1739. nickname=handle.split('@')[0]
  1740. domain=handle.split('@')[1]
  1741. archiveSubdir=None
  1742. if archiveDir:
  1743. if not os.path.isdir(archiveDir+'/accounts/'+handle):
  1744. os.mkdir(archiveDir+'/accounts/'+handle)
  1745. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
  1746. os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
  1747. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
  1748. os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
  1749. archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
  1750. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  1751. 'inbox',archiveSubdir, \
  1752. maxPostsInBox)
  1753. if archiveDir:
  1754. archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
  1755. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  1756. 'outbox',archiveSubdir, \
  1757. maxPostsInBox)
  1758. def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str, \
  1759. baseDir: str, \
  1760. boxname: str,archiveDir: str, \
  1761. maxPostsInBox=256) -> None:
  1762. """Retain a maximum number of posts within the given box
  1763. Move any others to an archive directory
  1764. """
  1765. if boxname!='inbox' and boxname!='outbox':
  1766. return
  1767. if archiveDir:
  1768. if not os.path.isdir(archiveDir):
  1769. os.mkdir(archiveDir)
  1770. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1771. postsInBox=sorted(os.listdir(boxDir), reverse=False)
  1772. noOfPosts=len(postsInBox)
  1773. if noOfPosts<=maxPostsInBox:
  1774. return
  1775. for postFilename in postsInBox:
  1776. filePath = os.path.join(boxDir, postFilename)
  1777. if os.path.isfile(filePath):
  1778. if archiveDir:
  1779. repliesPath=filePath.replace('.json','.replies')
  1780. archivePath = os.path.join(archiveDir, postFilename)
  1781. os.rename(filePath,archivePath)
  1782. if os.path.isfile(repliesPath):
  1783. os.rename(repliesPath,archivePath)
  1784. else:
  1785. deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
  1786. noOfPosts -= 1
  1787. if noOfPosts <= maxPostsInBox:
  1788. break
  1789. def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
  1790. raw: bool,simple: bool,useTor: bool, \
  1791. port: int,httpPrefix: str, \
  1792. debug: bool,projectVersion: str) -> None:
  1793. """ This is really just for test purposes
  1794. """
  1795. session = createSession(domain,port,useTor)
  1796. personCache={}
  1797. cachedWebfingers={}
  1798. federationList=[]
  1799. domainFull=domain
  1800. if port:
  1801. if port!=80 and port!=443:
  1802. if ':' not in domain:
  1803. domainFull=domain+':'+str(port)
  1804. handle=httpPrefix+"://"+domainFull+"/@"+nickname
  1805. wfRequest = \
  1806. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1807. domain,projectVersion)
  1808. if not wfRequest:
  1809. sys.exit()
  1810. personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
  1811. getPersonBox(baseDir,session,wfRequest,personCache, \
  1812. projectVersion,httpPrefix,domain,'outbox')
  1813. wfResult = json.dumps(wfRequest, indent=4, sort_keys=True)
  1814. maxMentions=10
  1815. maxEmoji=10
  1816. maxAttachments=5
  1817. userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
  1818. maxAttachments,federationList, \
  1819. personCache,raw,simple,debug, \
  1820. projectVersion,httpPrefix,domain)
  1821. #print(str(userPosts))
  1822. def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
  1823. nickname: str,domain: str,port: int, \
  1824. followerUrl,updateCaps: [], \
  1825. sendThreads: [],postLog: [], \
  1826. cachedWebfingers: {},personCache: {}, \
  1827. federationList :[],debug :bool, \
  1828. projectVersion: str) -> int:
  1829. """When the capabilities for a follower are changed this
  1830. sends out an update. followerUrl is the actor of the follower.
  1831. """
  1832. updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
  1833. nickname,domain,port, \
  1834. followerUrl, \
  1835. updateCaps)
  1836. if not updateJson:
  1837. return 1
  1838. if debug:
  1839. pprint(updateJson)
  1840. print('DEBUG: sending capabilities update from '+ \
  1841. nickname+'@'+domain+' port '+ str(port) + \
  1842. ' to '+followerUrl)
  1843. clientToServer=False
  1844. followerNickname=getNicknameFromActor(followerUrl)
  1845. if not followerNickname:
  1846. print('WARN: unable to find nickname in '+followerUrl)
  1847. return 1
  1848. followerDomain,followerPort=getDomainFromActor(followerUrl)
  1849. return sendSignedJson(updateJson,session,baseDir, \
  1850. nickname,domain,port, \
  1851. followerNickname,followerDomain,followerPort, '', \
  1852. httpPrefix,True,clientToServer, \
  1853. federationList, \
  1854. sendThreads,postLog,cachedWebfingers, \
  1855. personCache,debug,projectVersion)
  1856. def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
  1857. postRepliesFilename: str,authorized: bool, \
  1858. repliesJson: {}) -> None:
  1859. """populate the items list with replies
  1860. """
  1861. repliesBoxes=['outbox','inbox']
  1862. with open(postRepliesFilename,'r') as repliesFile:
  1863. for messageId in repliesFile:
  1864. replyFound=False
  1865. # examine inbox and outbox
  1866. for boxname in repliesBoxes:
  1867. searchFilename= \
  1868. baseDir+ \
  1869. '/accounts/'+nickname+'@'+ \
  1870. domain+'/'+ \
  1871. boxname+'/'+ \
  1872. messageId.replace('\n','').replace('/','#')+'.json'
  1873. if os.path.isfile(searchFilename):
  1874. if authorized or \
  1875. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  1876. with open(searchFilename, 'r') as fp:
  1877. postJsonObject=commentjson.load(fp)
  1878. if postJsonObject['object'].get('cc'):
  1879. if authorized or \
  1880. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  1881. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  1882. repliesJson['orderedItems'].append(postJsonObject)
  1883. replyFound=True
  1884. else:
  1885. if authorized or \
  1886. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  1887. repliesJson['orderedItems'].append(postJsonObject)
  1888. replyFound=True
  1889. break
  1890. # if not in either inbox or outbox then examine the shared inbox
  1891. if not replyFound:
  1892. searchFilename= \
  1893. baseDir+ \
  1894. '/accounts/inbox@'+ \
  1895. domain+'/inbox/'+ \
  1896. messageId.replace('\n','').replace('/','#')+'.json'
  1897. if os.path.isfile(searchFilename):
  1898. if authorized or \
  1899. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  1900. # get the json of the reply and append it to the collection
  1901. with open(searchFilename, 'r') as fp:
  1902. postJsonObject=commentjson.load(fp)
  1903. if postJsonObject['object'].get('cc'):
  1904. if authorized or \
  1905. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  1906. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  1907. repliesJson['orderedItems'].append(postJsonObject)
  1908. else:
  1909. if authorized or \
  1910. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  1911. repliesJson['orderedItems'].append(postJsonObject)