Mercurial > public > finance-parser

annotate upload_document/app.py @ 10:2350662483a3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix minor bugs
author Dennis C. M. <dennis@denniscm.com>
date Thu, 08 Jun 2023 17:16:36 +0100
parents d15ccf5f1373
children d09dee7a86da
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
1 import json
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
2 import boto3
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
3
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
4
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
5 s3_client = boto3.client('s3')
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
6 dynamodb = boto3.resource('dynamodb')
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
7 table = dynamodb.Table('FinanceParser')
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
8
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
9
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
10 def lambda_handler(event, context):
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
11 event_msg = event['body']['message']
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
12
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
13 # Download file from s3
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
14 s3_client.download_file(
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
15 event_msg['bucketName'],
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
16 event_msg['objectKey'],
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
17 '/tmp/document.json'
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
18 )
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
19
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
20 with open('/tmp/document.json') as f:
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
21 doc = json.load(f)
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
22
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
23 for dateColumn, date in doc['dateColumns'].items():
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
24 for row_index, account in doc['data'].items():
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
25
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
26 try:
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
27 column_types = account['type']
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
28 except KeyError:
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
29 column_types = []
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
30
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
31 """
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
32 Given:
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
33 +------------------+------+------+
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
34 | ASSETS | 2020 | 2019 |
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
35 +------------------+------+------+
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
36 | ASSETS_ACCOUNT_1 | | |
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
37 +------------------+------+------+
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
38 | ASSETS_ACCOUNT_2 | | |
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
39 +------------------+------+------+
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
40
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
41 The following statement avoids getting `2020` as the value of `ASSETS`.
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
42 """
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
43
10
2350662483a3 fix minor bugs
Dennis C. M. <dennis@denniscm.com>
parents: 6
diff changeset
44 try:
2350662483a3 fix minor bugs
Dennis C. M. <dennis@denniscm.com>
parents: 6
diff changeset
45 account_value = account[dateColumn]
2350662483a3 fix minor bugs
Dennis C. M. <dennis@denniscm.com>
parents: 6
diff changeset
46 except KeyError:
2350662483a3 fix minor bugs
Dennis C. M. <dennis@denniscm.com>
parents: 6
diff changeset
47 account_value = ''
2350662483a3 fix minor bugs
Dennis C. M. <dennis@denniscm.com>
parents: 6
diff changeset
48
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
49 if 'COLUMN_HEADER' in column_types and date == account_value:
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
50 account_value = ''
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
51
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
52 with table.batch_writer() as batch:
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
53
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
54 # pk -> item_type#company_ticker
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
55 # sk -> date#row_index
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
56
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
57 batch.put_item(
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
58 Item={
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
59 'pk': f"balance#{event_msg['companyTicker']}",
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
60 'sk': f'{date}#{row_index}',
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
61 'account_name': account['1'],
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
62 'account_value': account_value,
6
d15ccf5f1373 fix bug clean_text
Dennis C. M. <dennis@denniscm.com>
parents: 4
diff changeset
63 'column_types': column_types,
d15ccf5f1373 fix bug clean_text
Dennis C. M. <dennis@denniscm.com>
parents: 4
diff changeset
64 'format': doc['format']
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
65 }
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
66 )
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
67
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
68 # pk -> item_type#company_ticker
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
69 # sk -> date#filename
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
70
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
71 table.put_item(
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
72 Item={
6
d15ccf5f1373 fix bug clean_text
Dennis C. M. <dennis@denniscm.com>
parents: 4
diff changeset
73 'pk': f"file#balance#{event_msg['companyTicker']}",
4
9005b7590008 state machine working
Dennis C. M. <dennis@denniscm.com>
parents: 3
diff changeset
74 'sk': f"{date}#{event_msg['objectKey'].replace('processed/', '')}"
3
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
75 }
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
76 )
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
77
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
78 return {
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
79 "statusCode": 200,
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
80 "body": json.dumps({
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
81 "message": "ok"
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
82 }),
2e5f3664f3e4 documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
diff changeset
83 }