[ Oct 14 06:23:16 Enabled. ]
[ Oct 14 06:23:16 Rereading configuration. ]
[ Oct 14 06:23:18 Rereading configuration. ]
[ Oct 14 06:23:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:20.93684277Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:20.9387703Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:20.93878122Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:20.96874958Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:21.0799432Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.042769378Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57772"}
{"msg":"region 8ff6448c-3fee-4848-b169-94eceff9ac1f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.071049206Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.075283847Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions","method":"POST","req_id":"da51fbb6-4d09-40c1-a4d8-6ea50d9ecee4","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":8784,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.075313858Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.084632347Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.084684769Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8ff6448c-3fee-4848-b169-94eceff9ac1f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.17270624Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"8ff6448c-3fee-4848-b169-94eceff9ac1f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8ff6448c-3fee-4848-b169-94eceff9ac1f\" \"--data\" \"/data/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.176690314Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"8ff6448c-3fee-4848-b169-94eceff9ac1f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.308459943Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"8ff6448c-3fee-4848-b169-94eceff9ac1f","component":"worker"}
{"msg":"region 8ff6448c-3fee-4848-b169-94eceff9ac1f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.308535695Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.308707501Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.311116725Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.314697027Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.33252767Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.335850763Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.344018326Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.344096139Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.34413919Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.344189752Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.348215327Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.354004496Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:57.444642639Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions","method":"POST","req_id":"031c9bb1-5e41-4915-b0a3-938df10dea3b","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.408329373Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","method":"GET","req_id":"e98db655-49e6-46ea-9fb2-3ef44f7dd84e","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.411083289Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"dataset is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.422460062Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"region 8ff6448c-3fee-4848-b169-94eceff9ac1f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.432199904Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.43237075Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.432455732Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","method":"DELETE","req_id":"727630f6-2bc6-4e9c-8b10-fa3c324c476d","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21396,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8ff6448c-3fee-4848-b169-94eceff9ac1f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.433546486Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.436001402Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","method":"GET","req_id":"37a9f0dd-1f3a-4119-b5db-cffc4c6666d3","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":159,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.438658135Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.449865382Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"8ff6448c-3fee-4848-b169-94eceff9ac1f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.512376932Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","method":"GET","req_id":"6379fe00-55bc-444b-9532-83bc049ca49b","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":739,"response_code":"200"}
{"msg":"region 8ff6448c-3fee-4848-b169-94eceff9ac1f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.517291735Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:23:58.582043594Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/8ff6448c-3fee-4848-b169-94eceff9ac1f","method":"GET","req_id":"10588d40-d0b9-4499-9889-53e2135389ec","remote_addr":"[fd00:1122:3344:101::b]:57772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.427591584Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38233"}
{"msg":"region 2ecd36af-a72f-4bf2-9d31-e054a4d0adc5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.428398259Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.428569334Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.428584134Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions","method":"POST","req_id":"95f19c1f-d9c2-4353-a745-6b4eb23eb1a1","remote_addr":"[fd00:1122:3344:101::a]:38233","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":821,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.43779097Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.437828631Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.498018729Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\" \"--data\" \"/data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.498067111Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.606253478Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"region":"2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","component":"worker"}
{"msg":"region 2ecd36af-a72f-4bf2-9d31-e054a4d0adc5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.606289599Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.606381602Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.607764305Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.613383769Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.628600741Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.637202698Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.637465327Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.637529118Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.63757412Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.637616831Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.643410531Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.651820952Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:24:38.720154902Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions","method":"POST","req_id":"ab3fcf0f-2088-45c7-8a57-72532683073f","remote_addr":"[fd00:1122:3344:101::a]:38233","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.811853385Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62567"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.812009459Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","method":"GET","req_id":"6fdb2df5-ab85-4071-a4cb-c36438e28522","remote_addr":"[fd00:1122:3344:101::a]:62567","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.814851428Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"dataset is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.824794356Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"snapshot is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5@448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.834152077Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\", \"448dd239-4155-4b5c-9bd1-3e9a174b6484\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.834215139Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.848825212Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5/snapshots/448dd239-4155-4b5c-9bd1-3e9a174b6484","method":"GET","req_id":"06b6eb26-5c6b-4453-b69c-d6a54cd65353","remote_addr":"[fd00:1122:3344:101::a]:62567","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":34629,"response_code":"200"}
{"msg":"path is \"/data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.852213877Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"dataset is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.861155144Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"snapshot is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5@448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.870412892Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\", \"448dd239-4155-4b5c-9bd1-3e9a174b6484\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.870439043Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2ecd36af-a72f-4bf2-9d31-e054a4d0adc5-448dd239-4155-4b5c-9bd1-3e9a174b6484 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.880223536Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"applying SMF actions for region 2ecd36af-a72f-4bf2-9d31-e054a4d0adc5 running snapshot 448dd239-4155-4b5c-9bd1-3e9a174b6484 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.880841955Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.880887417Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5/snapshots/448dd239-4155-4b5c-9bd1-3e9a174b6484/run","method":"POST","req_id":"e6f9320d-8aaf-4290-b02f-c7706786efb5","remote_addr":"[fd00:1122:3344:101::a]:62567","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28730,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5-448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.89033233Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5-448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.897029608Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2ecd36af-a72f-4bf2-9d31-e054a4d0adc5-448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910080203Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5/.zfs/snapshot/448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910362962Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910416203Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910460825Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910501136Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.910547847Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.91481156Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.919075382Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"worker"}
{"msg":"region 2ecd36af-a72f-4bf2-9d31-e054a4d0adc5 running snapshot 448dd239-4155-4b5c-9bd1-3e9a174b6484 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:00.919085382Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"path is \"/data/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:01.240201226Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"dataset is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:01.249478844Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"datafile"}
{"msg":"snapshot is oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5@448dd239-4155-4b5c-9bd1-3e9a174b6484","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:01.258815134Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_842cb99f-15a1-44f5-a61e-c2655fd482df/crucible/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5\", \"448dd239-4155-4b5c-9bd1-3e9a174b6484\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:01.258860305Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T06:28:01.268266657Z","hostname":"oxz_crucible_112f32d7-b608-4d77-a581-df0a8e4fb714","pid":21083,"uri":"/crucible/0/regions/2ecd36af-a72f-4bf2-9d31-e054a4d0adc5/snapshots/448dd239-4155-4b5c-9bd1-3e9a174b6484/run","method":"POST","req_id":"d3670ea6-baf7-43ba-b93c-58abf6116562","remote_addr":"[fd00:1122:3344:101::a]:62567","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28114,"response_code":"200"}