Skip to content

Commit a45308d

Browse files
lostlevelslostlevels
committed
Create new OriginId index, hint, and remove "type" field from indexes
that don't need it.
1 parent fb72260 commit a45308d

File tree

2 files changed

+54
-25
lines changed

2 files changed

+54
-25
lines changed

‎data/store/mongo/mongo_datum.go‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -91,16 +91,16 @@ func (d *DatumRepository) EnsureIndexes() error {
9191
},
9292
}),
9393
},
94-
{
95-
Keys: bson.D{
96-
{Key: "origin.id", Value: 1},
97-
{Key: "type", Value: 1},
98-
{Key: "deletedTime", Value: -1},
99-
{Key: "_active", Value: 1},
100-
},
101-
Options: options.Index().
102-
SetName("OriginId"),
103-
},
94+
// {
95+
// Keys: bson.D{
96+
// {Key: "origin.id", Value: 1},
97+
// {Key: "type", Value: 1},
98+
// {Key: "deletedTime", Value: -1},
99+
// {Key: "_active", Value: 1},
100+
// },
101+
// Options: options.Index().
102+
// SetName("OriginId"),
103+
// },
104104
{
105105
Keys: bson.D{
106106
{Key: "_userId", Value: 1},
@@ -124,12 +124,26 @@ func (d *DatumRepository) EnsureIndexes() error {
124124
Options: options.Index().
125125
SetName("UploadId"),
126126
},
127+
// {
128+
// Keys: bson.D{
129+
// {Key: "_userId", Value: 1},
130+
// {Key: "deviceId", Value: 1},
131+
// {Key: "type", Value: 1},
132+
// {Key: "_active", Value: 1},
133+
// {Key: "_deduplicator.hash", Value: 1},
134+
// },
135+
// Options: options.Index().
136+
// SetPartialFilterExpression(bson.D{
137+
// {Key: "_active", Value: true},
138+
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
139+
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
140+
// }).
141+
// SetName("DeduplicatorHash"),
142+
// },
127143
{
128144
Keys: bson.D{
129145
{Key: "_userId", Value: 1},
130146
{Key: "deviceId", Value: 1},
131-
{Key: "type", Value: 1},
132-
{Key: "_active", Value: 1},
133147
{Key: "_deduplicator.hash", Value: 1},
134148
},
135149
Options: options.Index().
@@ -138,7 +152,7 @@ func (d *DatumRepository) EnsureIndexes() error {
138152
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
139153
{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
140154
}).
141-
SetName("DeduplicatorHash"),
155+
SetName("DeduplicatorHashNoType"),
142156
},
143157
})
144158
}
@@ -168,7 +182,8 @@ func (d *DatumRepository) CreateDataSetData(ctx context.Context, dataSet *upload
168182
datum.SetDataSetID(dataSet.UploadID)
169183
datum.SetCreatedTime(&timestamp)
170184
datum.SetModifiedTime(&timestamp)
171-
datum.SetModifiedTime(&timestamp)
185+
}
186+
for _, datum := range dataSetData {
172187
insertData = append(insertData, mongo.NewInsertOneModel().SetDocument(datum))
173188
}
174189

@@ -255,7 +270,7 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
255270
}
256271
opts := options.Update()
257272
if hasOriginID {
258-
opts.SetHint("OriginId")
273+
opts.SetHint("UserIdOriginId")
259274
}
260275
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
261276
if err != nil {
@@ -299,7 +314,7 @@ func (d *DatumRepository) DeleteDataSetData(ctx context.Context, dataSet *upload
299314
}
300315
opts := options.Update()
301316
if hasOriginID {
302-
opts.SetHint("OriginId")
317+
opts.SetHint("UserIdOriginId")
303318
}
304319
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
305320
if err != nil {
@@ -332,7 +347,7 @@ func (d *DatumRepository) DestroyDeletedDataSetData(ctx context.Context, dataSet
332347
selector["deletedTime"] = bson.M{"$exists": true}
333348
opts := options.Delete()
334349
if hasOriginID {
335-
opts.SetHint("OriginId")
350+
opts.SetHint("UserIdOriginId")
336351
}
337352
changeInfo, err := d.DeleteMany(ctx, selector, opts)
338353
if err != nil {
@@ -407,7 +422,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
407422
"modifiedTime": timestamp,
408423
}
409424
unset := bson.M{}
410-
opts := options.Update().SetHint("DeduplicatorHash")
425+
opts := options.Update().SetHint("DeduplicatorHashNoType")
411426
updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
412427
}
413428

@@ -535,7 +550,6 @@ func validateAndTranslateSelectors(selectors *data.Selectors) (filter bson.M, ha
535550
if selector.ID != nil {
536551
selectorIDs = append(selectorIDs, *selector.ID)
537552
} else if selector.Origin != nil && selector.Origin.ID != nil {
538-
hasOriginID = true
539553
selectorOriginIDs = append(selectorOriginIDs, *selector.Origin.ID)
540554
}
541555
}
@@ -746,8 +760,7 @@ func (d *DatumRepository) populateLastUpload(ctx context.Context, userId string,
746760

747761
findOptions := options.Find().SetProjection(bson.M{"_id": 0, "modifiedTime": 1, "createdTime": 1})
748762
if lowerTimeBound, err := time.Parse(time.RFC3339, LowerTimeIndexRaw); err == nil && timeMin.After(lowerTimeBound) {
749-
// has blocking sort, but more selective so usually performs better.
750-
findOptions.SetHint("UserIdActiveTypeTimeModifiedTime")
763+
findOptions.SetHint("TestUserIdActiveTypeModifiedTimeTime")
751764
}
752765
findOptions.SetLimit(1)
753766
findOptions.SetSort(bson.D{{Key: "modifiedTime", Value: -1}})

‎data/store/mongo/mongo_test.go‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -331,17 +331,33 @@ var _ = Describe("Mongo", func() {
331331
{Key: "time", Value: bson.D{{Key: "$gt", Value: primitive.NewDateTimeFromTime(lowerTimeIndex)}}},
332332
}),
333333
}),
334+
// MatchFields(IgnoreExtras, Fields{
335+
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
336+
// "Name": Equal("OriginId"),
337+
// }),
334338
MatchFields(IgnoreExtras, Fields{
335-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
336-
"Name": Equal("OriginId"),
339+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "origin.id", "-deletedTime", "_active")),
340+
"Name": Equal("UserIdOriginId"),
341+
"PartialFilterExpression": Equal(bson.D{
342+
{Key: "origin.id", Value: bson.D{{Key: "$exists", Value: true}}},
343+
}),
337344
}),
338345
MatchFields(IgnoreExtras, Fields{
339346
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
340347
"Name": Equal("UploadId"),
341348
}),
349+
// MatchFields(IgnoreExtras, Fields{
350+
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
351+
// "Name": Equal("DeduplicatorHash"),
352+
// "PartialFilterExpression": Equal(bson.D{
353+
// {Key: "_active", Value: true},
354+
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
355+
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
356+
// }),
357+
// }),
342358
MatchFields(IgnoreExtras, Fields{
343-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
344-
"Name": Equal("DeduplicatorHash"),
359+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "_deduplicator.hash")),
360+
"Name": Equal("DeduplicatorHashNoType"),
345361
"PartialFilterExpression": Equal(bson.D{
346362
{Key: "_active", Value: true},
347363
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},

0 commit comments

Comments
 (0)