Skip to content

hsbench: add unordered listing mode #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mattbenjamin wants to merge 1 commit into
base: master
Choose a base branch
from
Open

hsbench: add unordered listing mode #15

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions hsbench.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,55 @@ func runBucketList(thread_num int, stats *Stats) {
atomic.AddInt64(&running_threads, -1)
}

func runUnorderedList(thread_num int, stats *Stats) {
svc := s3.New(session.New(), cfg)

for { /* buckets */
bucket_num := atomic.AddInt64(&op_counter, 1)
if bucket_num >= bucket_count {
atomic.AddInt64(&op_counter, -1)
break
}

var next_marker string

start := time.Now().UnixNano()

for { /* pages */
input := &s3.ListObjectsInput{
Bucket: &buckets[bucket_num],
MaxKeys: &max_keys,
Marker: &next_marker,
Copy link
Owner

@markhpc markhpc Aug 29, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I remember there was some issue with using marker with the V1 ListObjects which is ultimately why I switched to ListObjectsPages. Not sure if this is related to the issue I hit above?

}
req, out := svc.ListObjectsRequest(input)
// need to add 'allow-unordered=True' to params
// n.b., eq marker&max-keys=1000
req.HTTPRequest.URL.RawQuery =
req.HTTPRequest.URL.RawQuery +
"&allow-unordered=True"

err := req.Send()
if err != nil {
log.Printf("unordered list err", err, "out",
out.String())
break
}

/* XXX should this really be every page? */
end := time.Now().UnixNano()
stats.updateIntervals(thread_num)
stats.addOp(thread_num, 0, end-start)
start = time.Now().UnixNano()

if (next_marker == "") {
break;
}
} /* pages */
} /* buckets */
stats.finish(thread_num)
atomic.AddInt64(&running_threads, -1)
}

var cfg *aws.Config

func runBucketsInit(thread_num int, stats *Stats) {
Expand Down Expand Up @@ -774,6 +823,12 @@ func runWrapper(loop int, r rune) []OutputStats {
for n := 0; n < threads; n++ {
go runBucketList(n, &stats)
}
case 'u':
log.Printf("Running Loop %d BUCKET UNORDERED LIST TEST", loop)
stats = makeStats(loop, "UNORDERED LIST", threads, intervalNano)
Copy link
Owner

@markhpc markhpc Aug 29, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: trying to keep these short since the log lines are so big. :) Can we change it from "UNORDERED LIST" to "ULIST"?

for n := 0; n < threads; n++ {
go runUnorderedList(n, &stats)
}
case 'g':
log.Printf("Running Loop %d OBJECT GET TEST", loop)
stats = makeStats(loop, "GET", threads, intervalNano)
Expand Down Expand Up @@ -846,6 +901,7 @@ NOTES:
i: initialize buckets
p: put objects in buckets
l: list objects in buckets
u: list objects unordered (RGW extension)
g: get objects from buckets
d: delete objects from buckets

Expand Down Expand Up @@ -890,6 +946,7 @@ NOTES:
r != 'p' &&
r != 'g' &&
r != 'l' &&
r != 'u' &&
r != 'd' &&
r != 'x' {
s := fmt.Sprintf("Invalid mode '%s' passed to -m", string(r))
Expand Down