cameron.stream
[READ-ONLY] Mirror of https://github.com/just-cameron/mongo-hack.
stuff
author
Cameron Pfiffer
date
(Apr 20, 2024, 8:37 PM -0700)
commit
72bc900b
72bc900bc2e93d68612ce2052ea288ece61f8fc0
parent
f8a92196
f8a921965a76d09dc1a74674349d22149d78c01d
+2150
+12
cameron/Dockerfile
+12
cameron/Dockerfile
+209
cameron/EMBED.PY
+209
cameron/EMBED.PY
···
1
+
from pathlib import Path
2
+
from fastapi import Depends, HTTPException, status
3
+
from fastapi.responses import FileResponse
4
+
from modal import Stub, web_endpoint, Image, Secret, method, NetworkFileSystem
5
+
import os
6
+
from typing import Dict
7
+
import json
8
+
9
+
from starlette.requests import Request
10
+
11
+
APP_NAME = "mongo-hack"
12
+
13
+
stub = Stub(name=APP_NAME)
14
+
RESULTS_DIR = "/results"
15
+
results_volume = NetworkFileSystem.new().persisted(f"{APP_NAME}-results-vol")
16
+
17
+
image = (
18
+
Image.debian_slim(
19
+
python_version="3.10"
20
+
).pip_install(
21
+
"replicate",
22
+
"python-jose",
23
+
"requests",
24
+
"openai",
25
+
"pymongo",
26
+
"scikit-learn",
27
+
"numpy"
28
+
)
29
+
)
30
+
31
+
@stub.function(image=image, keep_warm=1, secrets=[Secret.from_name("mongo-hack-secret")])
32
+
@web_endpoint(method="POST")
33
+
def handle_request(
34
+
payload: Dict,
35
+
request: Request
36
+
):
37
+
38
+
import openai
39
+
client = openai.OpenAI(
40
+
api_key=os.getenv('FIREWORKS_API_KEY'),
41
+
base_url="https://api.fireworks.ai/inference/v1"
42
+
)
43
+
44
+
input_text=payload["text"]
45
+
46
+
summary_response = client.chat.completions.create(
47
+
model="accounts/fireworks/models/llama-v2-7b-chat",
48
+
messages=[{
49
+
"role": "user",
50
+
"content": f"create a funny title for the given text: {input_text}",
51
+
}],
52
+
)
53
+
print(summary_response.choices[0].message.content)
54
+
55
+
response = client.embeddings.create(
56
+
model="nomic-ai/nomic-embed-text-v1.5",
57
+
input=input_text,
58
+
dimensions=768,
59
+
)
60
+
print(len(response.data[0].embedding))
61
+
62
+
# And also go make a title
63
+
64
+
new_title = "" # T
65
+
66
+
from pymongo import MongoClient
67
+
client = MongoClient("mongodb+srv://cameron:testing123@mongo-hackathon.4lo2sdu.mongodb.net/")
68
+
db = client["mongo-hackathon"]
69
+
print(db)
70
+
71
+
collection = db["documents"]
72
+
73
+
74
+
document = {
75
+
# "_id": ObjectId("6624177242be2e1d80040517"),
76
+
"embedding": response.data[0].embedding,
77
+
"text": input_text,
78
+
"title": new_title
79
+
}
80
+
result = collection.insert_one(document)
81
+
print("Inserted document ID:", result.inserted_id)
82
+
83
+
84
+
# Please return all the crap that goes along with this one
85
+
# {
86
+
# "$vectorSearch": {
87
+
# "index": "<index-name>",
88
+
# "path": "<field-to-search>",
89
+
# "queryVector": [<array-of-numbers>],
90
+
# "numCandidates": <number-of-candidates>,
91
+
# "limit": <number-of-results>,
92
+
# "filter": {<filter-specification>}
93
+
# }
94
+
# }
95
+
96
+
# pipeline = [
97
+
# {
98
+
# "$vectorSearch": {
99
+
# "index": "vector_index",
100
+
# "vectorSearchQuery": {
101
+
# "path": "plot_embedding",
102
+
# "queryVector": response.data[0].embedding,
103
+
# "numCandidates": 1000,
104
+
# }
105
+
# }
106
+
# },
107
+
# {
108
+
# "$limit": 10
109
+
# }
110
+
# ]
111
+
112
+
# # # define pipeline
113
+
# pipeline = [
114
+
# {
115
+
# '$vectorSearch': {
116
+
# 'index': 'vector_index',
117
+
# 'path': 'plot_embedding',
118
+
# 'queryVector': response.data[0].embedding,
119
+
# 'numCandidates': 1000,
120
+
# 'limit': 10
121
+
# }
122
+
# }, {
123
+
# '$project': {
124
+
# '_id': 1,
125
+
# 'embedding': 1,
126
+
# 'text': 1,
127
+
# 'score': {
128
+
# '$meta': 'vectorSearchScore'
129
+
# }
130
+
# }
131
+
# }
132
+
# ]
133
+
134
+
def query_results(query_vec):
135
+
results = collection.aggregate([
136
+
{
137
+
'$vectorSearch': {
138
+
"index": "embedding_index",
139
+
"path": "embedding",
140
+
"queryVector": query_vec,
141
+
"numCandidates": 1000,
142
+
"limit": 100,
143
+
}
144
+
}
145
+
])
146
+
return results
147
+
148
+
# run pipeline
149
+
# result = collection.aggregate(pipeline)
150
+
results = query_results(response.data[0].embedding)
151
+
results_collected = [r for r in results]
152
+
print("reslts collected shape")
153
+
print(len(results_collected))
154
+
155
+
import numpy as np
156
+
from sklearn.decomposition import PCA
157
+
158
+
def smush(matrix):
159
+
pca = PCA(n_components=2)
160
+
print(matrix.shape)
161
+
smushed = pca.fit_transform(matrix)
162
+
return smushed
163
+
164
+
# Converts a matrix to a JSON with the format
165
+
# {
166
+
# "data": [
167
+
# {
168
+
# "x": x1,
169
+
# "y": y1,
170
+
# "text": text1
171
+
# }
172
+
# ...
173
+
# ]
174
+
# }
175
+
def bundle(texts, matrix):
176
+
smushed = smush(matrix)
177
+
return {
178
+
"data": [
179
+
{
180
+
"x": x,
181
+
"y": y,
182
+
"text": text
183
+
}
184
+
for x, y, text in zip(smushed[:, 0], smushed[:, 1], texts)
185
+
]
186
+
}
187
+
188
+
# Unpack things
189
+
texts = [t["text"] for t in results_collected]
190
+
embeddings = [t["embedding"] for t in results_collected]
191
+
192
+
print("texts length")
193
+
print(len(texts))
194
+
195
+
print("embeddings length")
196
+
print(len(embeddings))
197
+
198
+
matrix = np.array(embeddings) - response.data[0].embedding
199
+
200
+
# smushed = smush(matrix)
201
+
# return json.dumps({
202
+
# 'xs':smushed[:,0],
203
+
# 'ys':smushed[:,1],
204
+
# })
205
+
206
+
return json.dumps(bundle(texts, matrix))
207
+
# return len(response.data[0].embedding)
208
+
209
+
+1242
cameron/Manifest.toml
+1242
cameron/Manifest.toml
···
1
+
# This file is machine-generated - editing it directly is not advised
2
+
3
+
julia_version = "1.10.2"
4
+
manifest_format = "2.0"
5
+
project_hash = "f60cfc56572091c487914b73474d62d43c1a2d09"
6
+
7
+
[[deps.AbstractTrees]]
8
+
git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177"
9
+
uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
10
+
version = "0.4.5"
11
+
12
+
[[deps.Accessors]]
13
+
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "LinearAlgebra", "MacroTools", "Markdown", "Test"]
14
+
git-tree-sha1 = "c0d491ef0b135fd7d63cbc6404286bc633329425"
15
+
uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
16
+
version = "0.1.36"
17
+
18
+
[deps.Accessors.extensions]
19
+
AccessorsAxisKeysExt = "AxisKeys"
20
+
AccessorsIntervalSetsExt = "IntervalSets"
21
+
AccessorsStaticArraysExt = "StaticArrays"
22
+
AccessorsStructArraysExt = "StructArrays"
23
+
AccessorsUnitfulExt = "Unitful"
24
+
25
+
[deps.Accessors.weakdeps]
26
+
AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
27
+
IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
28
+
Requires = "ae029012-a4dd-5104-9daa-d747884805df"
29
+
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
30
+
StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
31
+
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
32
+
33
+
[[deps.Adapt]]
34
+
deps = ["LinearAlgebra", "Requires"]
35
+
git-tree-sha1 = "6a55b747d1812e699320963ffde36f1ebdda4099"
36
+
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
37
+
version = "4.0.4"
38
+
weakdeps = ["StaticArrays"]
39
+
40
+
[deps.Adapt.extensions]
41
+
AdaptStaticArraysExt = "StaticArrays"
42
+
43
+
[[deps.AliasTables]]
44
+
deps = ["Random"]
45
+
git-tree-sha1 = "ca95b2220ef440817963baa71525a8f2f4ae7f8f"
46
+
uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
47
+
version = "1.0.0"
48
+
49
+
[[deps.ArgCheck]]
50
+
git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4"
51
+
uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197"
52
+
version = "2.3.0"
53
+
54
+
[[deps.ArgTools]]
55
+
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
56
+
version = "1.1.1"
57
+
58
+
[[deps.ArnoldiMethod]]
59
+
deps = ["LinearAlgebra", "Random", "StaticArrays"]
60
+
git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6"
61
+
uuid = "ec485272-7323-5ecc-a04f-4719b315124d"
62
+
version = "0.4.0"
63
+
64
+
[[deps.Arpack]]
65
+
deps = ["Arpack_jll", "Libdl", "LinearAlgebra", "Logging"]
66
+
git-tree-sha1 = "9b9b347613394885fd1c8c7729bfc60528faa436"
67
+
uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97"
68
+
version = "0.5.4"
69
+
70
+
[[deps.Arpack_jll]]
71
+
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "OpenBLAS_jll", "Pkg"]
72
+
git-tree-sha1 = "5ba6c757e8feccf03a1554dfaf3e26b3cfc7fd5e"
73
+
uuid = "68821587-b530-5797-8361-c406ea357684"
74
+
version = "3.5.1+1"
75
+
76
+
[[deps.ArrayInterface]]
77
+
deps = ["Adapt", "LinearAlgebra", "SparseArrays", "SuiteSparse"]
78
+
git-tree-sha1 = "133a240faec6e074e07c31ee75619c90544179cf"
79
+
uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
80
+
version = "7.10.0"
81
+
82
+
[deps.ArrayInterface.extensions]
83
+
ArrayInterfaceBandedMatricesExt = "BandedMatrices"
84
+
ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices"
85
+
ArrayInterfaceCUDAExt = "CUDA"
86
+
ArrayInterfaceCUDSSExt = "CUDSS"
87
+
ArrayInterfaceChainRulesExt = "ChainRules"
88
+
ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore"
89
+
ArrayInterfaceReverseDiffExt = "ReverseDiff"
90
+
ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore"
91
+
ArrayInterfaceTrackerExt = "Tracker"
92
+
93
+
[deps.ArrayInterface.weakdeps]
94
+
BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
95
+
BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
96
+
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
97
+
CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e"
98
+
ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
99
+
GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527"
100
+
ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
101
+
StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
102
+
Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
103
+
104
+
[[deps.ArrayLayouts]]
105
+
deps = ["FillArrays", "LinearAlgebra"]
106
+
git-tree-sha1 = "33207a8be6267bc389d0701e97a9bce6a4de68eb"
107
+
uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
108
+
version = "1.9.2"
109
+
weakdeps = ["SparseArrays"]
110
+
111
+
[deps.ArrayLayouts.extensions]
112
+
ArrayLayoutsSparseArraysExt = "SparseArrays"
113
+
114
+
[[deps.Artifacts]]
115
+
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
116
+
117
+
[[deps.BangBang]]
118
+
deps = ["Accessors", "Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires"]
119
+
git-tree-sha1 = "490e739172eb18f762e68dc3b928cad2a077983a"
120
+
uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66"
121
+
version = "0.4.1"
122
+
123
+
[deps.BangBang.extensions]
124
+
BangBangChainRulesCoreExt = "ChainRulesCore"
125
+
BangBangDataFramesExt = "DataFrames"
126
+
BangBangStaticArraysExt = "StaticArrays"
127
+
BangBangStructArraysExt = "StructArrays"
128
+
BangBangTablesExt = "Tables"
129
+
BangBangTypedTablesExt = "TypedTables"
130
+
131
+
[deps.BangBang.weakdeps]
132
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
133
+
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
134
+
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
135
+
StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
136
+
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
137
+
TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9"
138
+
139
+
[[deps.Base64]]
140
+
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
141
+
142
+
[[deps.Baselet]]
143
+
git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e"
144
+
uuid = "9718e550-a3fa-408a-8086-8db961cd8217"
145
+
version = "0.1.1"
146
+
147
+
[[deps.BinaryProvider]]
148
+
deps = ["Libdl", "Logging", "SHA"]
149
+
git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
150
+
uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
151
+
version = "0.5.10"
152
+
153
+
[[deps.BitFlags]]
154
+
git-tree-sha1 = "2dc09997850d68179b69dafb58ae806167a32b1b"
155
+
uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35"
156
+
version = "0.1.8"
157
+
158
+
[[deps.BitIntegers]]
159
+
deps = ["Random"]
160
+
git-tree-sha1 = "a55462dfddabc34bc97d3a7403a2ca2802179ae6"
161
+
uuid = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1"
162
+
version = "0.3.1"
163
+
164
+
[[deps.Calculus]]
165
+
deps = ["LinearAlgebra"]
166
+
git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad"
167
+
uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9"
168
+
version = "0.5.1"
169
+
170
+
[[deps.Clustering]]
171
+
deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"]
172
+
git-tree-sha1 = "9ebb045901e9bbf58767a9f34ff89831ed711aae"
173
+
uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5"
174
+
version = "0.15.7"
175
+
176
+
[[deps.CodecLz4]]
177
+
deps = ["Lz4_jll", "TranscodingStreams"]
178
+
git-tree-sha1 = "b8aecef9f90530cf322a8386630ec18485c17991"
179
+
uuid = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
180
+
version = "0.4.3"
181
+
182
+
[[deps.CodecZlib]]
183
+
deps = ["TranscodingStreams", "Zlib_jll"]
184
+
git-tree-sha1 = "59939d8a997469ee05c4b4944560a820f9ba0d73"
185
+
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
186
+
version = "0.7.4"
187
+
188
+
[[deps.CodecZstd]]
189
+
deps = ["TranscodingStreams", "Zstd_jll"]
190
+
git-tree-sha1 = "23373fecba848397b1705f6183188a0c0bc86917"
191
+
uuid = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
192
+
version = "0.8.2"
193
+
194
+
[[deps.CommonSubexpressions]]
195
+
deps = ["MacroTools", "Test"]
196
+
git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
197
+
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
198
+
version = "0.3.0"
199
+
200
+
[[deps.Compat]]
201
+
deps = ["TOML", "UUIDs"]
202
+
git-tree-sha1 = "c955881e3c981181362ae4088b35995446298b80"
203
+
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
204
+
version = "4.14.0"
205
+
weakdeps = ["Dates", "LinearAlgebra"]
206
+
207
+
[deps.Compat.extensions]
208
+
CompatLinearAlgebraExt = "LinearAlgebra"
209
+
210
+
[[deps.CompilerSupportLibraries_jll]]
211
+
deps = ["Artifacts", "Libdl"]
212
+
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
213
+
version = "1.1.0+0"
214
+
215
+
[[deps.CompositionsBase]]
216
+
git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
217
+
uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
218
+
version = "0.1.2"
219
+
weakdeps = ["InverseFunctions"]
220
+
221
+
[deps.CompositionsBase.extensions]
222
+
CompositionsBaseInverseFunctionsExt = "InverseFunctions"
223
+
224
+
[[deps.ConcurrentUtilities]]
225
+
deps = ["Serialization", "Sockets"]
226
+
git-tree-sha1 = "6cbbd4d241d7e6579ab354737f4dd95ca43946e1"
227
+
uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb"
228
+
version = "2.4.1"
229
+
230
+
[[deps.ConstructionBase]]
231
+
deps = ["LinearAlgebra"]
232
+
git-tree-sha1 = "260fd2400ed2dab602a7c15cf10c1933c59930a2"
233
+
uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
234
+
version = "1.5.5"
235
+
236
+
[deps.ConstructionBase.extensions]
237
+
ConstructionBaseIntervalSetsExt = "IntervalSets"
238
+
ConstructionBaseStaticArraysExt = "StaticArrays"
239
+
240
+
[deps.ConstructionBase.weakdeps]
241
+
IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
242
+
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
243
+
244
+
[[deps.DataAPI]]
245
+
git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
246
+
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
247
+
version = "1.16.0"
248
+
249
+
[[deps.DataDeps]]
250
+
deps = ["HTTP", "Libdl", "Reexport", "SHA", "Scratch", "p7zip_jll"]
251
+
git-tree-sha1 = "8ae085b71c462c2cb1cfedcb10c3c877ec6cf03f"
252
+
uuid = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
253
+
version = "0.7.13"
254
+
255
+
[[deps.DataStructures]]
256
+
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
257
+
git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82"
258
+
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
259
+
version = "0.18.20"
260
+
261
+
[[deps.DataValueInterfaces]]
262
+
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
263
+
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
264
+
version = "1.0.0"
265
+
266
+
[[deps.Dates]]
267
+
deps = ["Printf"]
268
+
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
269
+
270
+
[[deps.DecFP]]
271
+
deps = ["DecFP_jll", "Printf", "Random", "SpecialFunctions"]
272
+
git-tree-sha1 = "4a10cec664e26d9d63597daf9e62147e79d636e3"
273
+
uuid = "55939f99-70c6-5e9b-8bb0-5071ed7d61fd"
274
+
version = "1.3.2"
275
+
276
+
[[deps.DecFP_jll]]
277
+
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
278
+
git-tree-sha1 = "e9a8da19f847bbfed4076071f6fef8665a30d9e5"
279
+
uuid = "47200ebd-12ce-5be5-abb7-8e082af23329"
280
+
version = "2.0.3+1"
281
+
282
+
[[deps.DefineSingletons]]
283
+
git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c"
284
+
uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52"
285
+
version = "0.1.2"
286
+
287
+
[[deps.DiffResults]]
288
+
deps = ["StaticArraysCore"]
289
+
git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621"
290
+
uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
291
+
version = "1.1.0"
292
+
293
+
[[deps.DiffRules]]
294
+
deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"]
295
+
git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272"
296
+
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
297
+
version = "1.15.1"
298
+
299
+
[[deps.Distances]]
300
+
deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
301
+
git-tree-sha1 = "66c4c81f259586e8f002eacebc177e1fb06363b0"
302
+
uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
303
+
version = "0.10.11"
304
+
305
+
[deps.Distances.extensions]
306
+
DistancesChainRulesCoreExt = "ChainRulesCore"
307
+
DistancesSparseArraysExt = "SparseArrays"
308
+
309
+
[deps.Distances.weakdeps]
310
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
311
+
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
312
+
313
+
[[deps.Distributed]]
314
+
deps = ["Random", "Serialization", "Sockets"]
315
+
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
316
+
317
+
[[deps.Distributions]]
318
+
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
319
+
git-tree-sha1 = "22c595ca4146c07b16bcf9c8bea86f731f7109d2"
320
+
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
321
+
version = "0.25.108"
322
+
323
+
[deps.Distributions.extensions]
324
+
DistributionsChainRulesCoreExt = "ChainRulesCore"
325
+
DistributionsDensityInterfaceExt = "DensityInterface"
326
+
DistributionsTestExt = "Test"
327
+
328
+
[deps.Distributions.weakdeps]
329
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
330
+
DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
331
+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
332
+
333
+
[[deps.DocStringExtensions]]
334
+
deps = ["LibGit2"]
335
+
git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d"
336
+
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
337
+
version = "0.9.3"
338
+
339
+
[[deps.Downloads]]
340
+
deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
341
+
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
342
+
version = "1.6.0"
343
+
344
+
[[deps.DualNumbers]]
345
+
deps = ["Calculus", "NaNMath", "SpecialFunctions"]
346
+
git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566"
347
+
uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74"
348
+
version = "0.6.8"
349
+
350
+
[[deps.ExceptionUnwrapping]]
351
+
deps = ["Test"]
352
+
git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a"
353
+
uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4"
354
+
version = "0.1.10"
355
+
356
+
[[deps.FNVHash]]
357
+
git-tree-sha1 = "d6de2c735a8bffce9bc481942dfa453cc815357e"
358
+
uuid = "5207ad80-27db-4d23-8732-fa0bd339ea89"
359
+
version = "0.1.0"
360
+
361
+
[[deps.FilePathsBase]]
362
+
deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"]
363
+
git-tree-sha1 = "9f00e42f8d99fdde64d40c8ea5d14269a2e2c1aa"
364
+
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
365
+
version = "0.9.21"
366
+
367
+
[[deps.FileWatching]]
368
+
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
369
+
370
+
[[deps.FillArrays]]
371
+
deps = ["LinearAlgebra"]
372
+
git-tree-sha1 = "bfe82a708416cf00b73a3198db0859c82f741558"
373
+
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
374
+
version = "1.10.0"
375
+
weakdeps = ["PDMats", "SparseArrays", "Statistics"]
376
+
377
+
[deps.FillArrays.extensions]
378
+
FillArraysPDMatsExt = "PDMats"
379
+
FillArraysSparseArraysExt = "SparseArrays"
380
+
FillArraysStatisticsExt = "Statistics"
381
+
382
+
[[deps.FiniteDiff]]
383
+
deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays"]
384
+
git-tree-sha1 = "2de436b72c3422940cbe1367611d137008af7ec3"
385
+
uuid = "6a86dc24-6348-571c-b903-95158fe2bd41"
386
+
version = "2.23.1"
387
+
388
+
[deps.FiniteDiff.extensions]
389
+
FiniteDiffBandedMatricesExt = "BandedMatrices"
390
+
FiniteDiffBlockBandedMatricesExt = "BlockBandedMatrices"
391
+
FiniteDiffStaticArraysExt = "StaticArrays"
392
+
393
+
[deps.FiniteDiff.weakdeps]
394
+
BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
395
+
BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
396
+
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
397
+
398
+
[[deps.ForwardDiff]]
399
+
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"]
400
+
git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad"
401
+
uuid = "f6369f11-7733-5829-9624-2563aa707210"
402
+
version = "0.10.36"
403
+
weakdeps = ["StaticArrays"]
404
+
405
+
[deps.ForwardDiff.extensions]
406
+
ForwardDiffStaticArraysExt = "StaticArrays"
407
+
408
+
[[deps.Future]]
409
+
deps = ["Random"]
410
+
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
411
+
412
+
[[deps.Graphs]]
413
+
deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"]
414
+
git-tree-sha1 = "3863330da5466410782f2bffc64f3d505a6a8334"
415
+
uuid = "86223c79-3864-5bf0-83f7-82e725a168b6"
416
+
version = "1.10.0"
417
+
418
+
[[deps.HTML_Entities]]
419
+
deps = ["StrTables"]
420
+
git-tree-sha1 = "c4144ed3bc5f67f595622ad03c0e39fa6c70ccc7"
421
+
uuid = "7693890a-d069-55fe-a829-b4a6d304f0ee"
422
+
version = "1.0.1"
423
+
424
+
[[deps.HTTP]]
425
+
deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
426
+
git-tree-sha1 = "8e59b47b9dc525b70550ca082ce85bcd7f5477cd"
427
+
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
428
+
version = "1.10.5"
429
+
430
+
[[deps.HypergeometricFunctions]]
431
+
deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
432
+
git-tree-sha1 = "f218fe3736ddf977e0e772bc9a586b2383da2685"
433
+
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
434
+
version = "0.3.23"
435
+
436
+
[[deps.Inflate]]
437
+
git-tree-sha1 = "ea8031dea4aff6bd41f1df8f2fdfb25b33626381"
438
+
uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9"
439
+
version = "0.1.4"
440
+
441
+
[[deps.InitialValues]]
442
+
git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3"
443
+
uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c"
444
+
version = "0.3.1"
445
+
446
+
[[deps.InlineStrings]]
447
+
deps = ["Parsers"]
448
+
git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461"
449
+
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
450
+
version = "1.4.0"
451
+
452
+
[[deps.InteractiveUtils]]
453
+
deps = ["Markdown"]
454
+
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
455
+
456
+
[[deps.InverseFunctions]]
457
+
deps = ["Test"]
458
+
git-tree-sha1 = "896385798a8d49a255c398bd49162062e4a4c435"
459
+
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
460
+
version = "0.1.13"
461
+
weakdeps = ["Dates"]
462
+
463
+
[deps.InverseFunctions.extensions]
464
+
DatesExt = "Dates"
465
+
466
+
[[deps.IrrationalConstants]]
467
+
git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2"
468
+
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
469
+
version = "0.2.2"
470
+
471
+
[[deps.IterativeSolvers]]
472
+
deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"]
473
+
git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b"
474
+
uuid = "42fd0dbc-a981-5370-80f2-aaf504508153"
475
+
version = "0.9.4"
476
+
477
+
[[deps.IteratorInterfaceExtensions]]
478
+
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
479
+
uuid = "82899510-4779-5014-852e-03e436cf321d"
480
+
version = "1.0.0"
481
+
482
+
[[deps.JLLWrappers]]
483
+
deps = ["Artifacts", "Preferences"]
484
+
git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
485
+
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
486
+
version = "1.5.0"
487
+
488
+
[[deps.JSON]]
489
+
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
490
+
git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a"
491
+
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
492
+
version = "0.21.4"
493
+
494
+
[[deps.JSON3]]
495
+
deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"]
496
+
git-tree-sha1 = "eb3edce0ed4fa32f75a0a11217433c31d56bd48b"
497
+
uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
498
+
version = "1.14.0"
499
+
500
+
[deps.JSON3.extensions]
501
+
JSON3ArrowExt = ["ArrowTypes"]
502
+
503
+
[deps.JSON3.weakdeps]
504
+
ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
505
+
506
+
[[deps.LLMTextAnalysis]]
507
+
deps = ["Clustering", "Distances", "Languages", "LinearAlgebra", "Logging", "MLJLinearModels", "PrecompileTools", "PromptingTools", "Random", "Snowball", "SparseArrays", "Statistics", "Tables", "UMAP", "WordTokenizers"]
508
+
git-tree-sha1 = "5e16fdfc2de205b8bba72c16142aac2de33cdfef"
509
+
uuid = "a88142f3-a164-49e9-925a-9408902b6922"
510
+
version = "0.5.0"
511
+
512
+
[deps.LLMTextAnalysis.extensions]
513
+
PlotlyJSLLMTextAnalysisExt = ["PlotlyJS"]
514
+
PlotsLLMTextAnalysisExt = ["Plots"]
515
+
516
+
[deps.LLMTextAnalysis.weakdeps]
517
+
PlotlyJS = "f0f68f2c-4968-5e81-91da-67840de0976a"
518
+
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
519
+
520
+
[[deps.LZO_jll]]
521
+
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
522
+
git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
523
+
uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
524
+
version = "2.10.1+0"
525
+
526
+
[[deps.Languages]]
527
+
deps = ["InteractiveUtils", "JSON", "RelocatableFolders"]
528
+
git-tree-sha1 = "0cf92ba8402f94c9f4db0ec156888ee8d299fcb8"
529
+
uuid = "8ef0a80b-9436-5d2c-a485-80b904378c43"
530
+
version = "0.4.6"
531
+
532
+
[[deps.LazyArrays]]
533
+
deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "MatrixFactorizations", "SparseArrays"]
534
+
git-tree-sha1 = "35079a6a869eecace778bcda8641f9a54ca3a828"
535
+
uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02"
536
+
version = "1.10.0"
537
+
weakdeps = ["StaticArrays"]
538
+
539
+
[deps.LazyArrays.extensions]
540
+
LazyArraysStaticArraysExt = "StaticArrays"
541
+
542
+
[[deps.LibCURL]]
543
+
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
544
+
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
545
+
version = "0.6.4"
546
+
547
+
[[deps.LibCURL_jll]]
548
+
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
549
+
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
550
+
version = "8.4.0+0"
551
+
552
+
[[deps.LibGit2]]
553
+
deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
554
+
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
555
+
556
+
[[deps.LibGit2_jll]]
557
+
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
558
+
uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
559
+
version = "1.6.4+0"
560
+
561
+
[[deps.LibSSH2_jll]]
562
+
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
563
+
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
564
+
version = "1.11.0+1"
565
+
566
+
[[deps.Libdl]]
567
+
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
568
+
569
+
[[deps.LightBSON]]
570
+
deps = ["DataStructures", "Dates", "DecFP", "FNVHash", "JSON3", "Sockets", "StructTypes", "Transducers", "UUIDs", "UnsafeArrays", "WeakRefStrings"]
571
+
git-tree-sha1 = "d4d5cc8209c57ad04b35071da39ee8a006a0d938"
572
+
uuid = "a4a7f996-b3a6-4de6-b9db-2fa5f350df41"
573
+
version = "0.2.17"
574
+
575
+
[[deps.LineSearches]]
576
+
deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf"]
577
+
git-tree-sha1 = "7bbea35cec17305fc70a0e5b4641477dc0789d9d"
578
+
uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
579
+
version = "7.2.0"
580
+
581
+
[[deps.LinearAlgebra]]
582
+
deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
583
+
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
584
+
585
+
[[deps.LinearMaps]]
586
+
deps = ["LinearAlgebra"]
587
+
git-tree-sha1 = "9948d6f8208acfebc3e8cf4681362b2124339e7e"
588
+
uuid = "7a12625a-238d-50fd-b39a-03d52299707e"
589
+
version = "3.11.2"
590
+
591
+
[deps.LinearMaps.extensions]
592
+
LinearMapsChainRulesCoreExt = "ChainRulesCore"
593
+
LinearMapsSparseArraysExt = "SparseArrays"
594
+
LinearMapsStatisticsExt = "Statistics"
595
+
596
+
[deps.LinearMaps.weakdeps]
597
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
598
+
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
599
+
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
600
+
601
+
[[deps.LogExpFunctions]]
602
+
deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
603
+
git-tree-sha1 = "18144f3e9cbe9b15b070288eef858f71b291ce37"
604
+
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
605
+
version = "0.3.27"
606
+
607
+
[deps.LogExpFunctions.extensions]
608
+
LogExpFunctionsChainRulesCoreExt = "ChainRulesCore"
609
+
LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables"
610
+
LogExpFunctionsInverseFunctionsExt = "InverseFunctions"
611
+
612
+
[deps.LogExpFunctions.weakdeps]
613
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
614
+
ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
615
+
InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
616
+
617
+
[[deps.Logging]]
618
+
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
619
+
620
+
[[deps.LoggingExtras]]
621
+
deps = ["Dates", "Logging"]
622
+
git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075"
623
+
uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
624
+
version = "1.0.3"
625
+
626
+
[[deps.LsqFit]]
627
+
deps = ["Distributions", "ForwardDiff", "LinearAlgebra", "NLSolversBase", "Printf", "StatsAPI"]
628
+
git-tree-sha1 = "40acc20cfb253cf061c1a2a2ea28de85235eeee1"
629
+
uuid = "2fda8390-95c7-5789-9bda-21331edee243"
630
+
version = "0.15.0"
631
+
632
+
[[deps.Lz4_jll]]
633
+
deps = ["Artifacts", "JLLWrappers", "Libdl"]
634
+
git-tree-sha1 = "6c26c5e8a4203d43b5497be3ec5d4e0c3cde240a"
635
+
uuid = "5ced341a-0733-55b8-9ab6-a4889d929147"
636
+
version = "1.9.4+0"
637
+
638
+
[[deps.MLJLinearModels]]
639
+
deps = ["DocStringExtensions", "IterativeSolvers", "LinearAlgebra", "LinearMaps", "MLJModelInterface", "Optim", "Parameters"]
640
+
git-tree-sha1 = "7f517fd840ca433a8fae673edb31678ff55d969c"
641
+
uuid = "6ee0df7b-362f-4a72-a706-9e79364fb692"
642
+
version = "0.10.0"
643
+
644
+
[[deps.MLJModelInterface]]
645
+
deps = ["Random", "ScientificTypesBase", "StatisticalTraits"]
646
+
git-tree-sha1 = "d2a45e1b5998ba3fdfb6cfe0c81096d4c7fb40e7"
647
+
uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
648
+
version = "1.9.6"
649
+
650
+
[[deps.MacroTools]]
651
+
deps = ["Markdown", "Random"]
652
+
git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df"
653
+
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
654
+
version = "0.5.13"
655
+
656
+
[[deps.Markdown]]
657
+
deps = ["Base64"]
658
+
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
659
+
660
+
[[deps.MatrixFactorizations]]
661
+
deps = ["ArrayLayouts", "LinearAlgebra", "Printf", "Random"]
662
+
git-tree-sha1 = "6731e0574fa5ee21c02733e397beb133df90de35"
663
+
uuid = "a3b82374-2e81-5b9e-98ce-41277c0e4c87"
664
+
version = "2.2.0"
665
+
666
+
[[deps.MbedTLS]]
667
+
deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"]
668
+
git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf"
669
+
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
670
+
version = "1.1.9"
671
+
672
+
[[deps.MbedTLS_jll]]
673
+
deps = ["Artifacts", "Libdl"]
674
+
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
675
+
version = "2.28.2+1"
676
+
677
+
[[deps.MemPool]]
678
+
deps = ["DataStructures", "Distributed", "Mmap", "Random", "Serialization", "Sockets", "Test"]
679
+
git-tree-sha1 = "d52799152697059353a8eac1000d32ba8d92aa25"
680
+
uuid = "f9f48841-c794-520a-933b-121f7ba6ed94"
681
+
version = "0.2.0"
682
+
683
+
[[deps.MicroCollections]]
684
+
deps = ["Accessors", "BangBang", "InitialValues"]
685
+
git-tree-sha1 = "44d32db644e84c75dab479f1bc15ee76a1a3618f"
686
+
uuid = "128add7d-3638-4c79-886c-908ea0c25c34"
687
+
version = "0.2.0"
688
+
689
+
[[deps.Missings]]
690
+
deps = ["DataAPI"]
691
+
git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
692
+
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
693
+
version = "1.2.0"
694
+
695
+
[[deps.Mmap]]
696
+
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
697
+
698
+
[[deps.MongoC_jll]]
699
+
deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll", "Zstd_jll", "snappy_jll"]
700
+
git-tree-sha1 = "5a0f9f14a8186eae48608ff7922ac0c00ff52cdc"
701
+
uuid = "90100e71-7732-535a-9be7-2e9affd1cfc1"
702
+
version = "1.25.1+0"
703
+
704
+
[[deps.Mongoc]]
705
+
deps = ["Dates", "DecFP", "MongoC_jll", "Serialization"]
706
+
git-tree-sha1 = "f47bf7ed9d9c1da0a632777ca7dc406e3ad5f923"
707
+
uuid = "4fe8b98c-fc19-5c23-8ec2-168ff83495f2"
708
+
version = "0.9.0"
709
+
710
+
[[deps.MozillaCACerts_jll]]
711
+
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
712
+
version = "2023.1.10"
713
+
714
+
[[deps.MultivariateStats]]
715
+
deps = ["Arpack", "LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI", "StatsBase"]
716
+
git-tree-sha1 = "68bf5103e002c44adfd71fea6bd770b3f0586843"
717
+
uuid = "6f286f6a-111f-5878-ab1e-185364afe411"
718
+
version = "0.10.2"
719
+
720
+
[[deps.NLSolversBase]]
721
+
deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"]
722
+
git-tree-sha1 = "a0b464d183da839699f4c79e7606d9d186ec172c"
723
+
uuid = "d41bc354-129a-5804-8e4c-c37616107c6c"
724
+
version = "7.8.3"
725
+
726
+
[[deps.NaNMath]]
727
+
deps = ["OpenLibm_jll"]
728
+
git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4"
729
+
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
730
+
version = "1.0.2"
731
+
732
+
[[deps.NearestNeighborDescent]]
733
+
deps = ["DataStructures", "Distances", "Graphs", "Random", "Reexport", "SparseArrays"]
734
+
git-tree-sha1 = "b7d4bd2ab58f0c3a001fd6eedc2e0aac8e278152"
735
+
uuid = "dd2c4c9e-a32f-5b2f-b342-08c2f244fce8"
736
+
version = "0.3.6"
737
+
738
+
[[deps.NearestNeighbors]]
739
+
deps = ["Distances", "StaticArrays"]
740
+
git-tree-sha1 = "ded64ff6d4fdd1cb68dfcbb818c69e144a5b2e4c"
741
+
uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce"
742
+
version = "0.4.16"
743
+
744
+
[[deps.NetworkOptions]]
745
+
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
746
+
version = "1.2.0"
747
+
748
+
[[deps.OpenAI]]
749
+
deps = ["Dates", "HTTP", "JSON3"]
750
+
git-tree-sha1 = "c66f597044ac6cd41cbf4b191d59abbaf2003d9f"
751
+
uuid = "e9f21f70-7185-4079-aca2-91159181367c"
752
+
version = "0.9.0"
753
+
754
+
[[deps.OpenBLAS_jll]]
755
+
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
756
+
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
757
+
version = "0.3.23+4"
758
+
759
+
[[deps.OpenLibm_jll]]
760
+
deps = ["Artifacts", "Libdl"]
761
+
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
762
+
version = "0.8.1+2"
763
+
764
+
[[deps.OpenSSL]]
765
+
deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"]
766
+
git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4"
767
+
uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
768
+
version = "1.4.3"
769
+
770
+
[[deps.OpenSSL_jll]]
771
+
deps = ["Artifacts", "JLLWrappers", "Libdl"]
772
+
git-tree-sha1 = "a12e56c72edee3ce6b96667745e6cbbe5498f200"
773
+
uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
774
+
version = "1.1.23+0"
775
+
776
+
[[deps.OpenSpecFun_jll]]
777
+
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
778
+
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
779
+
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
780
+
version = "0.5.5+0"
781
+
782
+
[[deps.Optim]]
783
+
deps = ["Compat", "FillArrays", "ForwardDiff", "LineSearches", "LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "PositiveFactorizations", "Printf", "SparseArrays", "StatsBase"]
784
+
git-tree-sha1 = "d9b79c4eed437421ac4285148fcadf42e0700e89"
785
+
uuid = "429524aa-4258-5aef-a3af-852621145aeb"
786
+
version = "1.9.4"
787
+
788
+
[deps.Optim.extensions]
789
+
OptimMOIExt = "MathOptInterface"
790
+
791
+
[deps.Optim.weakdeps]
792
+
MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
793
+
794
+
[[deps.OrderedCollections]]
795
+
git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
796
+
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
797
+
version = "1.6.3"
798
+
799
+
[[deps.PDMats]]
800
+
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
801
+
git-tree-sha1 = "949347156c25054de2db3b166c52ac4728cbad65"
802
+
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
803
+
version = "0.11.31"
804
+
805
+
[[deps.Parameters]]
806
+
deps = ["OrderedCollections", "UnPack"]
807
+
git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe"
808
+
uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a"
809
+
version = "0.12.3"
810
+
811
+
[[deps.Parquet]]
812
+
deps = ["CodecZlib", "MemPool", "ProtoBuf", "Snappy", "Thrift"]
813
+
git-tree-sha1 = "d8970214b128dcd470d9da2d850722247be1ad9b"
814
+
uuid = "626c502c-15b0-58ad-a749-f091afb673ae"
815
+
version = "0.3.2"
816
+
817
+
[[deps.Parquet2]]
818
+
deps = ["AbstractTrees", "BitIntegers", "CodecLz4", "CodecZlib", "CodecZstd", "DataAPI", "Dates", "DecFP", "FilePathsBase", "FillArrays", "JSON3", "LazyArrays", "LightBSON", "Mmap", "OrderedCollections", "PooledArrays", "PrecompileTools", "SentinelArrays", "Snappy", "StaticArrays", "TableOperations", "Tables", "Thrift2", "Transducers", "UUIDs", "WeakRefStrings"]
819
+
git-tree-sha1 = "a3ab6596d22fd461b1e4fd6dcd30b8a4f419d78d"
820
+
uuid = "98572fba-bba0-415d-956f-fa77e587d26d"
821
+
version = "0.2.21"
822
+
823
+
[[deps.Parsers]]
824
+
deps = ["Dates", "PrecompileTools", "UUIDs"]
825
+
git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821"
826
+
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
827
+
version = "2.8.1"
828
+
829
+
[[deps.Pkg]]
830
+
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
831
+
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
832
+
version = "1.10.0"
833
+
834
+
[[deps.PooledArrays]]
835
+
deps = ["DataAPI", "Future"]
836
+
git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3"
837
+
uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
838
+
version = "1.4.3"
839
+
840
+
[[deps.PositiveFactorizations]]
841
+
deps = ["LinearAlgebra"]
842
+
git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20"
843
+
uuid = "85a6dd25-e78a-55b7-8502-1745935b8125"
844
+
version = "0.2.4"
845
+
846
+
[[deps.PrecompileTools]]
847
+
deps = ["Preferences"]
848
+
git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
849
+
uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
850
+
version = "1.2.1"
851
+
852
+
[[deps.Preferences]]
853
+
deps = ["TOML"]
854
+
git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
855
+
uuid = "21216c6a-2e73-6563-6e65-726566657250"
856
+
version = "1.4.3"
857
+
858
+
[[deps.Printf]]
859
+
deps = ["Unicode"]
860
+
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
861
+
862
+
[[deps.PromptingTools]]
863
+
deps = ["AbstractTrees", "Base64", "HTTP", "JSON3", "Logging", "OpenAI", "Pkg", "PrecompileTools", "Preferences", "Random", "Test"]
864
+
git-tree-sha1 = "f48828b20ada03d1af953296eae2658321f59731"
865
+
uuid = "670122d1-24a8-4d70-bfce-740807c42192"
866
+
version = "0.15.0"
867
+
weakdeps = ["LinearAlgebra", "Markdown", "SparseArrays"]
868
+
869
+
[deps.PromptingTools.extensions]
870
+
MarkdownPromptingToolsExt = ["Markdown"]
871
+
RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"]
872
+
873
+
[[deps.ProtoBuf]]
874
+
git-tree-sha1 = "51b74991da46594fb411a715e7e092bef50b99ff"
875
+
uuid = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
876
+
version = "0.8.0"
877
+
878
+
[[deps.QuadGK]]
879
+
deps = ["DataStructures", "LinearAlgebra"]
880
+
git-tree-sha1 = "9b23c31e76e333e6fb4c1595ae6afa74966a729e"
881
+
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
882
+
version = "2.9.4"
883
+
884
+
[[deps.REPL]]
885
+
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
886
+
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
887
+
888
+
[[deps.Random]]
889
+
deps = ["SHA"]
890
+
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
891
+
892
+
[[deps.RecipesBase]]
893
+
deps = ["PrecompileTools"]
894
+
git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff"
895
+
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
896
+
version = "1.3.4"
897
+
898
+
[[deps.Reexport]]
899
+
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
900
+
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
901
+
version = "1.2.2"
902
+
903
+
[[deps.RelocatableFolders]]
904
+
deps = ["SHA", "Scratch"]
905
+
git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864"
906
+
uuid = "05181044-ff0b-4ac5-8273-598c1e38db00"
907
+
version = "1.0.1"
908
+
909
+
[[deps.Requires]]
910
+
deps = ["UUIDs"]
911
+
git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
912
+
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
913
+
version = "1.3.0"
914
+
915
+
[[deps.Rmath]]
916
+
deps = ["Random", "Rmath_jll"]
917
+
git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b"
918
+
uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
919
+
version = "0.7.1"
920
+
921
+
[[deps.Rmath_jll]]
922
+
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
923
+
git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da"
924
+
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
925
+
version = "0.4.0+0"
926
+
927
+
[[deps.SHA]]
928
+
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
929
+
version = "0.7.0"
930
+
931
+
[[deps.ScientificTypesBase]]
932
+
git-tree-sha1 = "a8e18eb383b5ecf1b5e6fc237eb39255044fd92b"
933
+
uuid = "30f210dd-8aff-4c5f-94ba-8e64358c1161"
934
+
version = "3.0.0"
935
+
936
+
[[deps.Scratch]]
937
+
deps = ["Dates"]
938
+
git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386"
939
+
uuid = "6c6a2e73-6563-6170-7368-637461726353"
940
+
version = "1.2.1"
941
+
942
+
[[deps.SentinelArrays]]
943
+
deps = ["Dates", "Random"]
944
+
git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f"
945
+
uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
946
+
version = "1.4.1"
947
+
948
+
[[deps.Serialization]]
949
+
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
950
+
951
+
[[deps.Setfield]]
952
+
deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"]
953
+
git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac"
954
+
uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46"
955
+
version = "1.1.1"
956
+
957
+
[[deps.SharedArrays]]
958
+
deps = ["Distributed", "Mmap", "Random", "Serialization"]
959
+
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
960
+
961
+
[[deps.SimpleBufferStream]]
962
+
git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"
963
+
uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7"
964
+
version = "1.1.0"
965
+
966
+
[[deps.SimpleTraits]]
967
+
deps = ["InteractiveUtils", "MacroTools"]
968
+
git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231"
969
+
uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d"
970
+
version = "0.9.4"
971
+
972
+
[[deps.Snappy]]
973
+
deps = ["BinaryProvider", "Libdl", "Random", "Test"]
974
+
git-tree-sha1 = "25620a91907972a05863941d6028791c2613888e"
975
+
uuid = "59d4ed8c-697a-5b28-a4c7-fe95c22820f9"
976
+
version = "0.3.0"
977
+
978
+
[[deps.Snowball]]
979
+
deps = ["Languages", "Snowball_jll", "WordTokenizers"]
980
+
git-tree-sha1 = "8b466b16804ab8687f8d3a1b5312a0aa1b7d8b64"
981
+
uuid = "fb8f903a-0164-4e73-9ffe-431110250c3b"
982
+
version = "0.1.1"
983
+
984
+
[[deps.Snowball_jll]]
985
+
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
986
+
git-tree-sha1 = "6ff3a185a583dca7265cbfcaae1da16aa3b6a962"
987
+
uuid = "88f46535-a3c0-54f4-998e-4320a1339f51"
988
+
version = "2.2.0+0"
989
+
990
+
[[deps.Sockets]]
991
+
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
992
+
993
+
[[deps.SortingAlgorithms]]
994
+
deps = ["DataStructures"]
995
+
git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085"
996
+
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
997
+
version = "1.2.1"
998
+
999
+
[[deps.SparseArrays]]
1000
+
deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
1001
+
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
1002
+
version = "1.10.0"
1003
+
1004
+
[[deps.SpecialFunctions]]
1005
+
deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
1006
+
git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d"
1007
+
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
1008
+
version = "2.3.1"
1009
+
1010
+
[deps.SpecialFunctions.extensions]
1011
+
SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
1012
+
1013
+
[deps.SpecialFunctions.weakdeps]
1014
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
1015
+
1016
+
[[deps.SplittablesBase]]
1017
+
deps = ["Setfield", "Test"]
1018
+
git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5"
1019
+
uuid = "171d559e-b47b-412a-8079-5efa626c420e"
1020
+
version = "0.1.15"
1021
+
1022
+
[[deps.StaticArrays]]
1023
+
deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"]
1024
+
git-tree-sha1 = "bf074c045d3d5ffd956fa0a461da38a44685d6b2"
1025
+
uuid = "90137ffa-7385-5640-81b9-e52037218182"
1026
+
version = "1.9.3"
1027
+
1028
+
[deps.StaticArrays.extensions]
1029
+
StaticArraysChainRulesCoreExt = "ChainRulesCore"
1030
+
StaticArraysStatisticsExt = "Statistics"
1031
+
1032
+
[deps.StaticArrays.weakdeps]
1033
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
1034
+
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1035
+
1036
+
[[deps.StaticArraysCore]]
1037
+
git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d"
1038
+
uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
1039
+
version = "1.4.2"
1040
+
1041
+
[[deps.StatisticalTraits]]
1042
+
deps = ["ScientificTypesBase"]
1043
+
git-tree-sha1 = "30b9236691858e13f167ce829490a68e1a597782"
1044
+
uuid = "64bff920-2084-43da-a3e6-9bb72801c0c9"
1045
+
version = "3.2.0"
1046
+
1047
+
[[deps.Statistics]]
1048
+
deps = ["LinearAlgebra", "SparseArrays"]
1049
+
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1050
+
version = "1.10.0"
1051
+
1052
+
[[deps.StatsAPI]]
1053
+
deps = ["LinearAlgebra"]
1054
+
git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed"
1055
+
uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
1056
+
version = "1.7.0"
1057
+
1058
+
[[deps.StatsBase]]
1059
+
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
1060
+
git-tree-sha1 = "5cf7606d6cef84b543b483848d4ae08ad9832b21"
1061
+
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
1062
+
version = "0.34.3"
1063
+
1064
+
[[deps.StatsFuns]]
1065
+
deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
1066
+
git-tree-sha1 = "cef0472124fab0695b58ca35a77c6fb942fdab8a"
1067
+
uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
1068
+
version = "1.3.1"
1069
+
1070
+
[deps.StatsFuns.extensions]
1071
+
StatsFunsChainRulesCoreExt = "ChainRulesCore"
1072
+
StatsFunsInverseFunctionsExt = "InverseFunctions"
1073
+
1074
+
[deps.StatsFuns.weakdeps]
1075
+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
1076
+
InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
1077
+
1078
+
[[deps.StrTables]]
1079
+
deps = ["Dates"]
1080
+
git-tree-sha1 = "5998faae8c6308acc25c25896562a1e66a3bb038"
1081
+
uuid = "9700d1a9-a7c8-5760-9816-a99fda30bb8f"
1082
+
version = "1.0.1"
1083
+
1084
+
[[deps.StructTypes]]
1085
+
deps = ["Dates", "UUIDs"]
1086
+
git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70"
1087
+
uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
1088
+
version = "1.10.0"
1089
+
1090
+
[[deps.SuiteSparse]]
1091
+
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
1092
+
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
1093
+
1094
+
[[deps.SuiteSparse_jll]]
1095
+
deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
1096
+
uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
1097
+
version = "7.2.1+1"
1098
+
1099
+
[[deps.TOML]]
1100
+
deps = ["Dates"]
1101
+
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
1102
+
version = "1.0.3"
1103
+
1104
+
[[deps.TableOperations]]
1105
+
deps = ["SentinelArrays", "Tables", "Test"]
1106
+
git-tree-sha1 = "e383c87cf2a1dc41fa30c093b2a19877c83e1bc1"
1107
+
uuid = "ab02a1b2-a7df-11e8-156e-fb1833f50b87"
1108
+
version = "1.2.0"
1109
+
1110
+
[[deps.TableTraits]]
1111
+
deps = ["IteratorInterfaceExtensions"]
1112
+
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
1113
+
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
1114
+
version = "1.0.1"
1115
+
1116
+
[[deps.Tables]]
1117
+
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"]
1118
+
git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d"
1119
+
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
1120
+
version = "1.11.1"
1121
+
1122
+
[[deps.Tar]]
1123
+
deps = ["ArgTools", "SHA"]
1124
+
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
1125
+
version = "1.10.0"
1126
+
1127
+
[[deps.Test]]
1128
+
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
1129
+
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1130
+
1131
+
[[deps.Thrift]]
1132
+
deps = ["BinaryProvider", "Distributed", "Sockets"]
1133
+
git-tree-sha1 = "c3dd01c6067985a77fef761839203838ac12825b"
1134
+
uuid = "8d9c9c80-f77e-5080-9541-c6f69d204e22"
1135
+
version = "0.6.2"
1136
+
1137
+
[[deps.Thrift2]]
1138
+
deps = ["MacroTools", "OrderedCollections", "PrecompileTools"]
1139
+
git-tree-sha1 = "00d618714271f283ea3829ab058d5e5bd1847f85"
1140
+
uuid = "9be31aac-5446-47db-bfeb-416acd2e4415"
1141
+
version = "0.1.4"
1142
+
1143
+
[[deps.TranscodingStreams]]
1144
+
git-tree-sha1 = "71509f04d045ec714c4748c785a59045c3736349"
1145
+
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
1146
+
version = "0.10.7"
1147
+
weakdeps = ["Random", "Test"]
1148
+
1149
+
[deps.TranscodingStreams.extensions]
1150
+
TestExt = ["Test", "Random"]
1151
+
1152
+
[[deps.Transducers]]
1153
+
deps = ["Accessors", "Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "SplittablesBase", "Tables"]
1154
+
git-tree-sha1 = "47e516e2eabd0cf1304cd67839d9a85d52dd659d"
1155
+
uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999"
1156
+
version = "0.4.81"
1157
+
1158
+
[deps.Transducers.extensions]
1159
+
TransducersBlockArraysExt = "BlockArrays"
1160
+
TransducersDataFramesExt = "DataFrames"
1161
+
TransducersLazyArraysExt = "LazyArrays"
1162
+
TransducersOnlineStatsBaseExt = "OnlineStatsBase"
1163
+
TransducersReferenceablesExt = "Referenceables"
1164
+
1165
+
[deps.Transducers.weakdeps]
1166
+
BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e"
1167
+
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
1168
+
LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02"
1169
+
OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338"
1170
+
Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e"
1171
+
1172
+
[[deps.UMAP]]
1173
+
deps = ["Arpack", "Distances", "LinearAlgebra", "LsqFit", "NearestNeighborDescent", "Random", "SparseArrays"]
1174
+
git-tree-sha1 = "df15e2580e56acb1bde25d2ac36460d22f0f57f2"
1175
+
uuid = "c4f8c510-2410-5be4-91d7-4fbaeb39457e"
1176
+
version = "0.1.11"
1177
+
1178
+
[[deps.URIs]]
1179
+
git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b"
1180
+
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
1181
+
version = "1.5.1"
1182
+
1183
+
[[deps.UUIDs]]
1184
+
deps = ["Random", "SHA"]
1185
+
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
1186
+
1187
+
[[deps.UnPack]]
1188
+
git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
1189
+
uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
1190
+
version = "1.0.2"
1191
+
1192
+
[[deps.Unicode]]
1193
+
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
1194
+
1195
+
[[deps.UnsafeArrays]]
1196
+
git-tree-sha1 = "e7f1c67ba99ac6df440de191fa4d5cbfcbdddcd1"
1197
+
uuid = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6"
1198
+
version = "1.0.5"
1199
+
1200
+
[[deps.WeakRefStrings]]
1201
+
deps = ["DataAPI", "InlineStrings", "Parsers"]
1202
+
git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23"
1203
+
uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
1204
+
version = "1.4.2"
1205
+
1206
+
[[deps.WordTokenizers]]
1207
+
deps = ["DataDeps", "HTML_Entities", "StrTables", "Unicode"]
1208
+
git-tree-sha1 = "01dd4068c638da2431269f49a5964bf42ff6c9d2"
1209
+
uuid = "796a5d58-b03d-544a-977e-18100b691f6e"
1210
+
version = "0.5.6"
1211
+
1212
+
[[deps.Zlib_jll]]
1213
+
deps = ["Libdl"]
1214
+
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
1215
+
version = "1.2.13+1"
1216
+
1217
+
[[deps.Zstd_jll]]
1218
+
deps = ["Artifacts", "JLLWrappers", "Libdl"]
1219
+
git-tree-sha1 = "e678132f07ddb5bfa46857f0d7620fb9be675d3b"
1220
+
uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
1221
+
version = "1.5.6+0"
1222
+
1223
+
[[deps.libblastrampoline_jll]]
1224
+
deps = ["Artifacts", "Libdl"]
1225
+
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
1226
+
version = "5.8.0+1"
1227
+
1228
+
[[deps.nghttp2_jll]]
1229
+
deps = ["Artifacts", "Libdl"]
1230
+
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
1231
+
version = "1.52.0+1"
1232
+
1233
+
[[deps.p7zip_jll]]
1234
+
deps = ["Artifacts", "Libdl"]
1235
+
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
1236
+
version = "17.4.0+2"
1237
+
1238
+
[[deps.snappy_jll]]
1239
+
deps = ["Artifacts", "JLLWrappers", "LZO_jll", "Libdl", "Zlib_jll"]
1240
+
git-tree-sha1 = "ab27636e7c8222f14b9318a983fcd89cf130d419"
1241
+
uuid = "fe1e1685-f7be-5f59-ac9f-4ca204017dfd"
1242
+
version = "1.1.10+0"
+8
cameron/Project.toml
+8
cameron/Project.toml
···
1
+
[deps]
2
+
Clustering = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5"
3
+
LLMTextAnalysis = "a88142f3-a164-49e9-925a-9408902b6922"
4
+
Mongoc = "4fe8b98c-fc19-5c23-8ec2-168ff83495f2"
5
+
MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411"
6
+
Parquet = "626c502c-15b0-58ad-a749-f091afb673ae"
7
+
Parquet2 = "98572fba-bba0-415d-956f-fa77e587d26d"
8
+
PromptingTools = "670122d1-24a8-4d70-bfce-740807c42192"
cameron/__pycache__/embed.cpython-310.pyc
cameron/__pycache__/embed.cpython-310.pyc
This is a binary file and will not be displayed.
+305
cameron/embed.py
+305
cameron/embed.py
···
1
+
from pathlib import Path
2
+
from fastapi import Depends, HTTPException, status
3
+
from fastapi.responses import FileResponse
4
+
from modal import Stub, web_endpoint, Image, Secret, method, NetworkFileSystem
5
+
import os
6
+
from typing import Dict
7
+
import json
8
+
9
+
from starlette.requests import Request
10
+
11
+
APP_NAME = "mongo-hack"
12
+
13
+
stub = Stub(name=APP_NAME)
14
+
RESULTS_DIR = "/results"
15
+
results_volume = NetworkFileSystem.from_name(f"{APP_NAME}-results-vol", create_if_missing=True)
16
+
# results_volume = NetworkFileSystem.new().persisted(f"{APP_NAME}-results-vol")
17
+
18
+
image = (
19
+
Image.debian_slim(
20
+
python_version="3.10"
21
+
).pip_install(
22
+
"replicate",
23
+
"python-jose",
24
+
"requests",
25
+
"openai",
26
+
"pymongo",
27
+
"scikit-learn",
28
+
"numpy"
29
+
)
30
+
)
31
+
32
+
@stub.function(image=image, keep_warm=1, secrets=[Secret.from_name("mongo-hack-secret")])
33
+
@web_endpoint(method="POST")
34
+
def handle_request(
35
+
payload: Dict,
36
+
request: Request
37
+
):
38
+
39
+
from pymongo import MongoClient
40
+
client = MongoClient("mongodb+srv://cameron:testing123@mongo-hackathon.4lo2sdu.mongodb.net/")
41
+
db = client["mongo-hackathon"]
42
+
print(db)
43
+
44
+
collection = db["documents"]
45
+
input_text=payload["text"]
46
+
47
+
import openai
48
+
fireworks_client = openai.OpenAI(
49
+
api_key=os.getenv('FIREWORKS_API_KEY'),
50
+
base_url="https://api.fireworks.ai/inference/v1"
51
+
)
52
+
53
+
# Check if input text is already in the database
54
+
existing_documents = [d for d in collection.find({"text": input_text})]
55
+
document_exists = len(existing_documents) != 0
56
+
57
+
print("Document exists:", document_exists)
58
+
59
+
# If we already have the document, get the embedding.
60
+
if document_exists:
61
+
embedding = existing_documents[0]["embedding"]
62
+
else:
63
+
response = fireworks_client.embeddings.create(
64
+
model="nomic-ai/nomic-embed-text-v1.5",
65
+
input=input_text,
66
+
dimensions=768,
67
+
)
68
+
69
+
embedding = response.data[0].embedding
70
+
71
+
print(len(embedding))
72
+
73
+
# Okay -- if our document exists, check if it has a title, and if it does, use it. Otherwise, generate a title.
74
+
# if document_exists:
75
+
# print("Document exists, checking for title")
76
+
# if "title" in existing_documents[0] and existing_documents[0]["title"] != "":
77
+
# print("Title exists, using it")
78
+
# new_title = existing_documents[0]["title"]
79
+
# print("Old title:", new_title)
80
+
# else:
81
+
# print("Title doesn't exist, generating it")
82
+
# new_title = fire_function__call_generate_title(fireworks_client, input_text)
83
+
# print("New title:", new_title)
84
+
85
+
# # Update the document
86
+
# collection.update_one({"_id": existing_documents[0]["_id"]}, {"$set": {"title": new_title}})
87
+
# else:
88
+
# new_title = fire_function__call_generate_title(fireworks_client, input_text)
89
+
new_title = ""
90
+
91
+
document = {
92
+
# "_id": ObjectId("6624177242be2e1d80040517"),
93
+
"embedding": embedding,
94
+
"text": input_text,
95
+
"title": new_title
96
+
}
97
+
98
+
if not document_exists:
99
+
result = collection.insert_one(document)
100
+
print("Inserted document ID:", result.inserted_id)
101
+
102
+
def query_results(query_vec):
103
+
results = collection.aggregate([
104
+
{
105
+
'$vectorSearch': {
106
+
"index": "embedding_index",
107
+
"path": "embedding",
108
+
"queryVector": query_vec,
109
+
"numCandidates": 1000,
110
+
"limit": 100,
111
+
}
112
+
}
113
+
])
114
+
return results
115
+
116
+
# run pipeline
117
+
results = query_results(embedding)
118
+
results_collected = [r for r in results]
119
+
print("reslts collected shape")
120
+
print(len(results_collected))
121
+
122
+
import numpy as np
123
+
from sklearn.decomposition import PCA
124
+
from sklearn.preprocessing import MinMaxScaler
125
+
126
+
def smush(matrix):
127
+
pca = PCA(n_components=2)
128
+
print(matrix.shape)
129
+
smushed = pca.fit_transform(matrix)
130
+
return smushed
131
+
132
+
# Converts a matrix to a JSON with the format
133
+
# {
134
+
# "data": [
135
+
# {
136
+
# "x": x1,
137
+
# "y": y1,
138
+
# "text": text1
139
+
# }
140
+
# ...
141
+
# ]
142
+
# }
143
+
def bundle(texts, titles, matrix):
144
+
smushed = smush(matrix)
145
+
146
+
# Normalize the matrix
147
+
scaler = MinMaxScaler(feature_range=(-1, 1))
148
+
smushed = scaler.fit_transform(matrix)
149
+
150
+
return {
151
+
"data": [
152
+
{
153
+
"x": x,
154
+
"y": y,
155
+
"text": text,
156
+
"title": title
157
+
}
158
+
for x, y, text, title in zip(smushed[:, 0], smushed[:, 1], texts, titles)
159
+
]
160
+
}
161
+
162
+
# Unpack things
163
+
texts = [t["text"] for t in results_collected]
164
+
embeddings = [t["embedding"] for t in results_collected]
165
+
166
+
print("texts length")
167
+
print(len(texts))
168
+
169
+
print("embeddings length")
170
+
print(len(embeddings))
171
+
172
+
matrix = np.array(embeddings)
173
+
return json.dumps(bundle(texts, [""] * len(texts), matrix))
174
+
175
+
176
+
def fire_function_call(client):
177
+
178
+
messages = [
179
+
{
180
+
"role": "system",
181
+
"content": "You are a very creative assistant with access to functions. Use them if required."
182
+
},
183
+
{
184
+
"role": "user",
185
+
"content": "Select a super random topic and create a very short title and super detailed long text description for the topic"
186
+
}
187
+
]
188
+
189
+
tools = [
190
+
{
191
+
"type": "function",
192
+
"function": {
193
+
"name": "get_title_text",
194
+
"description": "Get the random text and title.",
195
+
"parameters": {
196
+
"type": "object",
197
+
"properties": {
198
+
"random_text": {
199
+
"type": "string",
200
+
"description": "random text",
201
+
},
202
+
"title": {
203
+
"type": "string",
204
+
"description": "title for random text."
205
+
},
206
+
},
207
+
"required": ["random_text", "title"],
208
+
},
209
+
},
210
+
}
211
+
]
212
+
213
+
chat_completion = client.chat.completions.create(
214
+
model="accounts/fireworks/models/fw-function-call-34b-v0",
215
+
messages=messages,
216
+
tools=tools,
217
+
tool_choice="auto",
218
+
temperature=0.8
219
+
)
220
+
221
+
print(repr(chat_completion.choices[0].message.model_dump()))
222
+
223
+
message_data = chat_completion.choices[0].message.model_dump()
224
+
225
+
# Assuming message_data now contains a dictionary with the 'tool_calls' key
226
+
tool_calls = message_data['tool_calls']
227
+
228
+
# Extract the function arguments from the first tool call
229
+
function_args_str = tool_calls[0]['function']['arguments']
230
+
231
+
# Parse the JSON string in function arguments
232
+
function_args = json.loads(function_args_str)
233
+
234
+
# Extract 'random_text' and 'title' values
235
+
random_text = function_args['random_text']
236
+
title = function_args['title']
237
+
238
+
print("Random Text:", random_text)
239
+
print("Title:", title)
240
+
241
+
return random_text, title
242
+
243
+
244
+
245
+
246
+
def fire_function__call_generate_title(client, text):
247
+
248
+
messages = [
249
+
{
250
+
"role": "system",
251
+
"content": "You are an assistant good at creative titles with access to functions. Please use the get_title function in your response."
252
+
},
253
+
{
254
+
"role": "user",
255
+
"content": f"Create a very short title for the input text: {text}"
256
+
}
257
+
]
258
+
259
+
tools = [
260
+
{
261
+
"type": "function",
262
+
"function": {
263
+
"name": "get_title",
264
+
"description": "Generate a short title.",
265
+
"parameters": {
266
+
"type": "object",
267
+
"properties": {
268
+
"title": {
269
+
"type": "string",
270
+
"description": "title for input text."
271
+
},
272
+
},
273
+
"required": ["title"],
274
+
},
275
+
},
276
+
}
277
+
]
278
+
279
+
chat_completion = client.chat.completions.create(
280
+
model="accounts/fireworks/models/fw-function-call-34b-v0",
281
+
messages=messages,
282
+
tools=tools,
283
+
tool_choice="auto",
284
+
temperature=0.1
285
+
)
286
+
287
+
print(repr(chat_completion.choices[0].message.model_dump()))
288
+
289
+
message_data = chat_completion.choices[0].message.model_dump()
290
+
291
+
# Assuming message_data now contains a dictionary with the 'tool_calls' key
292
+
tool_calls = message_data['tool_calls']
293
+
294
+
# Extract the function arguments from the first tool call
295
+
function_args_str = tool_calls[0]['function']['arguments']
296
+
297
+
# Parse the JSON string in function arguments
298
+
function_args = json.loads(function_args_str)
299
+
300
+
# Extract 'random_text' and 'title' values
301
+
title = function_args['title']
302
+
303
+
print("Title:", title)
304
+
305
+
return title
+3
cameron/env.jl
+3
cameron/env.jl
···
1
+
ENV["FIREWORKS_API_KEY"] = "bm38HrH82R4Qc2AdindG4xSHlVb9AlJMcNWyoRAH0akh1Mze"
2
+
ENV["SSL_CERT_DIR"] = "/etc/ssl/certs/"
3
+
ENV["MONGO_CONNSTRING"] = "mongodb+srv://cameron:testing123@mongo-hackathon.4lo2sdu.mongodb.net/?retryWrites=true&w=majority&appName=mongo-hackathon?tlsCAFile=/etc/ssl/certs/ca-certificates.crt"
+12
cameron/fireworks.py
+12
cameron/fireworks.py
···
1
+
import openai
2
+
3
+
client = openai.OpenAI(
4
+
base_url = "https://api.fireworks.ai/inference/v1",
5
+
api_key="bm38HrH82R4Qc2AdindG4xSHlVb9AlJMcNWyoRAH0akh1Mze",
6
+
)
7
+
response = client.embeddings.create(
8
+
model="nomic-ai/nomic-embed-text-v1.5",
9
+
input="search_document: Spiderman was a particularly entertaining movie with...",
10
+
)
11
+
12
+
print(response)
+145
cameron/go.jl
+145
cameron/go.jl
···
1
+
using Mongoc
2
+
using PromptingTools
3
+
using Clustering
4
+
using MultivariateStats
5
+
using Statistics
6
+
using Plots
7
+
8
+
# get env shit
9
+
include("env.jl")
10
+
11
+
connection_string = ENV["MONGO_CONNSTRING"]
12
+
client = Mongoc.Client(connection_string)
13
+
@assert Mongoc.ping(client)["ok"] == 1
14
+
db = client["mongo-hackathon"]
15
+
collection = db["documents"]
16
+
17
+
# Convenience function to get embedding
18
+
function embed_fireworks(text::AbstractString; model="nomic-ai/nomic-embed-text-v1.5")
19
+
return PromptingTools.aiembed(
20
+
PromptingTools.FireworksOpenAISchema(),
21
+
text,
22
+
model=model
23
+
)
24
+
end
25
+
26
+
function embed(text::AbstractString; model="mxbai-embed-large")
27
+
return PromptingTools.aiembed(
28
+
PromptingTools.OllamaSchema(),
29
+
text,
30
+
model=model
31
+
).content |> Array
32
+
end
33
+
34
+
# Random text generator
35
+
function makeprompt()
36
+
#
37
+
new_prompt = aigenerate(
38
+
PromptingTools.OllamaSchema(),
39
+
"Give me an interesting fact. Please respond only with the fact.",
40
+
model="mistral:latest"
41
+
).content
42
+
43
+
println(new_prompt)
44
+
45
+
return new_prompt
46
+
47
+
# Ask more
48
+
# response = aigenerate(
49
+
# PromptingTools.OllamaSchema(),
50
+
# "Please tell me more about $new_prompt",
51
+
# model="gemma:2b"
52
+
# ).content
53
+
54
+
# # Make a title for this text
55
+
# title = aigenerate(
56
+
# PromptingTools.OllamaSchema(),
57
+
# "Please give me a title for this text: $new_prompt",
58
+
# model="gemma:2b"
59
+
# ).content
60
+
61
+
# return (title, new_prompt, response)
62
+
end
63
+
64
+
# bson convenience function
65
+
bson(x) = Mongoc.BSON(x)
66
+
67
+
# Function to upload text to mongo
68
+
function upload(text::AbstractString; model="nomic-embed-text:latest")
69
+
# Check if the text already exists
70
+
exists = Mongoc.find_one(collection, bson(Dict("text" => text)))
71
+
if !isnothing(exists)
72
+
@info "Doc exists, skipping insert"
73
+
return exists
74
+
end
75
+
76
+
# Get embedding
77
+
embedding = embed(text; model=model)
78
+
79
+
# Start a document
80
+
doc = Dict(
81
+
"text" => text,
82
+
"embedding" => embedding
83
+
)
84
+
85
+
# Insert into collection
86
+
push!(collection, bson(doc))
87
+
end
88
+
89
+
# upload("cameron")
90
+
# upload("gus")
91
+
# upload("anand")
92
+
# upload("dog")
93
+
# upload("cat")
94
+
95
+
for i in 1:1000
96
+
upload(makeprompt())
97
+
end
98
+
99
+
100
+
101
+
102
+
103
+
# {
104
+
# "$vectorSearch": {
105
+
# "index": "<index-name>",
106
+
# "path": "<field-to-search>",
107
+
# "queryVector": [<array-of-numbers>],
108
+
# "numCandidates": <number-of-candidates>,
109
+
# "limit": <number-of-results>,
110
+
# "filter": {<filter-specification>}
111
+
# }
112
+
# }
113
+
114
+
# qvec = "cameron"
115
+
# query_embedding = embed(qvec)
116
+
# query_doc = bson(Dict(
117
+
# "\$vectorSearch" => Dict(
118
+
# "queryVector" => [query_embedding],
119
+
# )
120
+
# ))
121
+
122
+
# println(query_doc)
123
+
124
+
# for c in Mongoc.aggregate(collection, query_doc)
125
+
# println(c)
126
+
# end
127
+
128
+
# docs = collect(collection)
129
+
embeddings = reduce(hcat, map(identity, doc["embedding"]) for doc in docs)
130
+
texts = [doc["text"] for doc in docs]
131
+
M = fit(PCA, embeddings; maxoutdim=2)
132
+
smushed = predict(M, embeddings)
133
+
scanned = Clustering.dbscan(smushed, 0.5)
134
+
135
+
cluster_docs = Dict{Int,Vector{String}}()
136
+
for (assignment, text) = zip(scanned.assignments, texts)
137
+
if !haskey(cluster_docs, assignment)
138
+
cluster_docs[assignment] = String[]
139
+
end
140
+
push!(cluster_docs[assignment], text)
141
+
end
142
+
143
+
# Now we want to transform this matrix so that the first row is the origin
144
+
transformation_matrix = smushed
145
+
+43
cameron/parquet-load.py
+43
cameron/parquet-load.py
···
1
+
import pandas as pd
2
+
import glob
3
+
import requests
4
+
import json
5
+
import os # Make sure to import os at the top of your file
6
+
7
+
def load_parquet_files(folder_path):
8
+
# Expand the user's home directory
9
+
folder_path = os.path.expanduser(folder_path)
10
+
# Create a pattern to match all parquet files in the folder
11
+
pattern = f"{folder_path}/*.parquet"
12
+
# Use glob to find all files matching the pattern
13
+
parquet_files = glob.glob(pattern)
14
+
# Load and concatenate all parquet files into a single DataFrame
15
+
df = pd.concat([pd.read_parquet(file) for file in parquet_files[0:1]], ignore_index=True)
16
+
return df
17
+
18
+
def post_text_to_endpoint(df, url):
19
+
# Iterate over each row in the DataFrame
20
+
for index, row in df.iterrows():
21
+
# Extract text from the current row
22
+
text = row['text']
23
+
# Prepare the data for POST request
24
+
data = json.dumps({"text": text})
25
+
headers = {'Content-Type': 'application/json'}
26
+
# Send POST request to the specified URL
27
+
response = requests.post(url, data=data, headers=headers)
28
+
# Check if the request was successful
29
+
if response.status_code == 200:
30
+
print(f"Successfully posted text from row {index}")
31
+
else:
32
+
print(f"Failed to post text from row {index}. Status code: {response.status_code}")
33
+
34
+
# Define the folder path and URL
35
+
folder_path = "~/code/tiny-strange-textbooks"
36
+
url = "https://cpfiffer--mongo-hack-handle-request-dev.modal.run/"
37
+
38
+
# Load parquet files into a DataFrame
39
+
df = load_parquet_files(folder_path)
40
+
print(df)
41
+
42
+
# Post text to the specified endpoint
43
+
post_text_to_endpoint(df, url)
+7
cameron/run.sh
+7
cameron/run.sh
+35
cameron/server.py
+35
cameron/server.py
···
1
+
# ENV["FIREWORKS_API_KEY"] = "bm38HrH82R4Qc2AdindG4xSHlVb9AlJMcNWyoRAH0akh1Mze"
2
+
# ENV["SSL_CERT_DIR"] = "/etc/ssl/certs/"
3
+
# ENV["MONGO_CONNSTRING"] = "mongodb+srv://cameron:testing123@mongo-hackathon.4lo2sdu.mongodb.net/?retryWrites=true&w=majority&appName=mongo-hackathon?tlsCAFile=/etc/ssl/certs/ca-certificates.crt"
4
+
5
+
6
+
import os
7
+
from flask import Flask, request, jsonify
8
+
from pymongo import MongoClient
9
+
10
+
app = Flask(__name__)
11
+
12
+
# Connect to MongoDB
13
+
mongo_connstring = os.environ["MONGO_CONNSTRING"]
14
+
client = MongoClient(mongo_connstring)
15
+
db = client.data
16
+
17
+
@app.route('/text', methods=['POST'])
18
+
def handle_text():
19
+
# Get the text from the request
20
+
text = request.json['text']
21
+
22
+
# TODO: Process the text and generate x and y
23
+
x = "example_x"
24
+
y = "example_y"
25
+
26
+
# Return the response as JSON
27
+
response = {
28
+
"documents": text,
29
+
"x": x,
30
+
"y": y
31
+
}
32
+
return jsonify(response)
33
+
34
+
if __name__ == '__main__':
35
+
app.run()
+79
cameron/smush.py
+79
cameron/smush.py
···
1
+
import numpy as np
2
+
import matplotlib.pyplot as plt
3
+
from sklearn.preprocessing import MinMaxScaler
4
+
5
+
# Define the dimensions of the matrix
6
+
n = 1000
7
+
k = 2 # Use 2D data for easier visualization
8
+
9
+
# Create the matrix
10
+
data = np.random.randn(n, k)
11
+
12
+
# Re-center the matrix around the first row
13
+
recentered_data = data - data[0]
14
+
15
+
# Define the rotation angle in degrees
16
+
rotation_angle_deg = 45
17
+
# Convert rotation angle to radians
18
+
rotation_angle_rad = np.deg2rad(rotation_angle_deg)
19
+
20
+
# Define scaling factors for each axis
21
+
scaling_x = 1.5
22
+
scaling_y = 0.75
23
+
24
+
# Define shear factors for each axis
25
+
shear_x = 0.5
26
+
shear_y = 0.2
27
+
28
+
# Create a 2D rotation matrix
29
+
rotation_matrix = np.array([
30
+
[np.cos(rotation_angle_rad), -np.sin(rotation_angle_rad)],
31
+
[np.sin(rotation_angle_rad), np.cos(rotation_angle_rad)]
32
+
])
33
+
34
+
# Create a 2D scaling matrix
35
+
scaling_matrix = np.array([
36
+
[scaling_x, 0],
37
+
[0, scaling_y]
38
+
])
39
+
40
+
# Create a 2D shear matrix
41
+
shear_matrix = np.array([
42
+
[1, shear_x],
43
+
[shear_y, 1]
44
+
])
45
+
46
+
# Combine rotation, scaling, and shear matrices
47
+
transformation_matrix = np.dot(rotation_matrix, np.dot(scaling_matrix, shear_matrix))
48
+
49
+
# Apply the transformation matrix to the recentered data
50
+
transformed_data = np.dot(recentered_data, transformation_matrix)
51
+
52
+
# Normalize the transformed data to be between -1 and 1
53
+
scaler = MinMaxScaler(feature_range=(-1, 1))
54
+
normalized_transformed_data = scaler.fit_transform(transformed_data)
55
+
56
+
# Plotting the original and normalized transformed data
57
+
plt.figure(figsize=(10, 5))
58
+
59
+
# Plot original data (re-centered)
60
+
plt.subplot(1, 2, 1)
61
+
plt.scatter(recentered_data[:, 0], recentered_data[:, 1], color='b', label='Original Data')
62
+
plt.scatter(recentered_data[0, 0], recentered_data[0, 1], color='r', marker='*', s=100, label='First Row')
63
+
plt.title('Recentered Data')
64
+
plt.xlabel('X')
65
+
plt.ylabel('Y')
66
+
plt.legend()
67
+
68
+
# Plot normalized transformed data
69
+
plt.subplot(1, 2, 2)
70
+
plt.scatter(normalized_transformed_data[:, 0], normalized_transformed_data[:, 1], color='r', label='Normalized Transformed Data')
71
+
plt.scatter(normalized_transformed_data[0, 0], normalized_transformed_data[0, 1], color='b', marker='*', s=100, label='First Row')
72
+
plt.title('Normalized Transformed Data')
73
+
plt.xlabel('X')
74
+
plt.ylabel('Y')
75
+
plt.legend()
76
+
77
+
# Show the plots
78
+
plt.tight_layout()
79
+
plt.show()
+50
cameron/upload.jl
+50
cameron/upload.jl
···
1
+
using HTTP
2
+
using JSON3
3
+
using Parquet2, DataFrames
4
+
# df = DataFrame(read_parquet(path))
5
+
6
+
function load_parquet_files(folder_path::String)
7
+
# load the parquet files
8
+
parquet_files = filter(
9
+
x -> endswith(x, ".parquet"),
10
+
readdir(folder_path, join=true)
11
+
)
12
+
13
+
# Load and concatenate all parquet files into a single DataFrame
14
+
dfs = DataFrame[]
15
+
for file in parquet_files[1:10]
16
+
ds = Parquet2.Dataset(file)
17
+
df = DataFrame(ds; copycols=false)
18
+
push!(dfs, df)
19
+
end
20
+
return reduce(vcat, dfs)
21
+
end
22
+
23
+
function post_text_to_endpoint(df::DataFrame, url::String)
24
+
# Iterate over each row in the DataFrame
25
+
for row in eachrow(df)
26
+
# Extract text from the current row
27
+
text = row["text"]
28
+
# Prepare the data for POST request
29
+
data = JSON3.write(Dict("text" => text))
30
+
headers = Dict("Content-Type" => "application/json")
31
+
# Send POST request to the specified URL
32
+
response = HTTP.post(url, headers=headers, body=data)
33
+
# Check if the request was successful
34
+
if response.status == 200
35
+
println("Successfully posted text from row")
36
+
else
37
+
println("Failed to post text from row. Status code: $(response.status)")
38
+
end
39
+
end
40
+
end
41
+
42
+
# Define the folder path and URL
43
+
folder_path = joinpath(homedir(), "code/tiny-strange-textbooks")
44
+
url = "https://cpfiffer--mongo-hack-handle-request-dev.modal.run/"
45
+
46
+
# Load parquet files into a DataFrame
47
+
df = load_parquet_files(folder_path)
48
+
49
+
# Post text to the specified endpoint
50
+
post_text_to_endpoint(df, url)