[ Dec 18 22:13:59 Enabled. ]
[ Dec 18 22:13:59 Rereading configuration. ]
[ Dec 18 22:14:01 Rereading configuration. ]
[ Dec 18 22:14:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.834116268Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.838854431Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.838865701Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.854353115Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.928326056Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.950553187Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63290"}
{"msg":"region f0cf758d-2948-4338-9d33-bab047b6aaee state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.971365685Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.973741322Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.973762992Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"6c516434-df03-4a98-bfce-00110a994d9c","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":8720,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.983938742Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.983971283Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f0cf758d-2948-4338-9d33-bab047b6aaee\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f0cf758d-2948-4338-9d33-bab047b6aaee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.054051148Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"region":"f0cf758d-2948-4338-9d33-bab047b6aaee","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f0cf758d-2948-4338-9d33-bab047b6aaee\" \"--data\" \"/data/regions/f0cf758d-2948-4338-9d33-bab047b6aaee\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.054108349Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"region":"f0cf758d-2948-4338-9d33-bab047b6aaee","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.143676756Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"0f4448ab-7e82-4f9e-a371-5c086c06fb30","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.32891068Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"region":"f0cf758d-2948-4338-9d33-bab047b6aaee","component":"worker"}
{"msg":"region f0cf758d-2948-4338-9d33-bab047b6aaee state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.328954011Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.329042033Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.32941641Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.336046211Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.346484535Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.349367722Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.350099376Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.350161728Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.350204778Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.350244559Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.354320139Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.358225446Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.866834944Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"6104126d-255f-4dd1-921e-539ee77002a5","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.334070731Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","method":"GET","req_id":"1685ef85-c2df-403e-8d72-daf430dfdee2","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/f0cf758d-2948-4338-9d33-bab047b6aaee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.338248643Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"dataset is oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.348988894Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"region f0cf758d-2948-4338-9d33-bab047b6aaee state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.359537281Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.359670804Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.359734765Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","method":"DELETE","req_id":"831946a3-1992-45d6-b858-78c1ef84b24f","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21522,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.362398027Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","method":"GET","req_id":"3402a94c-7b23-4d6f-b7c9-6f62ba469767","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-f0cf758d-2948-4338-9d33-bab047b6aaee (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.364152951Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.368784573Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8d122e51-f03c-4659-8af4-3a5baca809c5/crucible/regions/f0cf758d-2948-4338-9d33-bab047b6aaee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.379021933Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"region":"f0cf758d-2948-4338-9d33-bab047b6aaee","component":"worker"}
{"msg":"region f0cf758d-2948-4338-9d33-bab047b6aaee state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.45020389Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.468302305Z","hostname":"oxz_crucible_387d3af0-07c5-480d-840e-4805c74ee7db","pid":23789,"uri":"/crucible/0/regions/f0cf758d-2948-4338-9d33-bab047b6aaee","method":"GET","req_id":"6d1bfb91-74a9-484f-bec1-90691a70bdc2","remote_addr":"[fd00:1122:3344:101::a]:63290","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":64,"response_code":"200"}