[ Jun 17 18:06:37 Enabled. ]
[ Jun 17 18:06:37 Rereading configuration. ]
[ Jun 17 18:06:39 Rereading configuration. ]
[ Jun 17 18:06:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:40.670990867Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:40.673772784Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:40.673793095Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:40.695848772Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:40.811119347Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.737269259Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42681"}
{"msg":"region 4433059c-4543-44e6-af5a-50bc706c8bc1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.775778356Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.783011852Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions","method":"POST","req_id":"75101b74-5d91-49de-8fe3-1701cadf1055","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17495,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.783039453Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.794858151Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.794890202Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4433059c-4543-44e6-af5a-50bc706c8bc1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4433059c-4543-44e6-af5a-50bc706c8bc1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.845961545Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"region":"4433059c-4543-44e6-af5a-50bc706c8bc1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4433059c-4543-44e6-af5a-50bc706c8bc1\" \"--data\" \"/data/regions/4433059c-4543-44e6-af5a-50bc706c8bc1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.847796229Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"region":"4433059c-4543-44e6-af5a-50bc706c8bc1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:27.973078268Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions","method":"POST","req_id":"ef6e2be5-cd3e-4859-8e59-501704b0aa8e","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.161140295Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"region":"4433059c-4543-44e6-af5a-50bc706c8bc1","component":"worker"}
{"msg":"region 4433059c-4543-44e6-af5a-50bc706c8bc1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.161180676Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.161310169Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.161740019Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.168894893Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.181674975Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.18683355Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.192715493Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.194991149Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.19505899Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.195108891Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.199954709Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.204270205Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:28.389256637Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions","method":"POST","req_id":"0ce49b41-cb32-427c-bc94-d3b231006150","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.637230302Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","method":"GET","req_id":"42e53da6-3a68-4276-a40a-3af8c46a6361","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/4433059c-4543-44e6-af5a-50bc706c8bc1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.640509582Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"dataset is oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.649804858Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"region 4433059c-4543-44e6-af5a-50bc706c8bc1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.658615713Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.658789427Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-4433059c-4543-44e6-af5a-50bc706c8bc1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.659833353Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.661116414Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","method":"DELETE","req_id":"a9f1d91f-08b3-4d6d-b63e-3187d5681b87","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21208,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.663925782Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","method":"GET","req_id":"dacf4790-f3dc-40ce-a26b-39951d05e7d2","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.663947793Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_605b7a32-ab28-4fc2-a860-9027f98c0954/crucible/regions/4433059c-4543-44e6-af5a-50bc706c8bc1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.672909321Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"region":"4433059c-4543-44e6-af5a-50bc706c8bc1","component":"worker"}
{"msg":"region 4433059c-4543-44e6-af5a-50bc706c8bc1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.724080776Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:51.74722935Z","hostname":"oxz_crucible_1e49a06e-2a0e-492e-ac3a-fbabb954acef","pid":22154,"uri":"/crucible/0/regions/4433059c-4543-44e6-af5a-50bc706c8bc1","method":"GET","req_id":"9603ba56-0391-4deb-9ad8-6297e018e6b1","remote_addr":"[fd00:1122:3344:101::a]:42681","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":53,"response_code":"200"}