[ Nov 26 23:31:55 Enabled. ]
[ Nov 26 23:31:55 Rereading configuration. ]
[ Nov 26 23:31:57 Rereading configuration. ]
[ Nov 26 23:32:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.677738158Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.682107359Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.682120739Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.696936727Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.756986818Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.597291527Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56006"}
{"msg":"region 4307145a-0ee7-44e1-a760-c298d88c1a95 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.641693572Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.646803459Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.651238141Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions","method":"POST","req_id":"976df2ef-2ef3-4d89-b018-4c2d0248ab82","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16944,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.659841921Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.659863341Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4307145a-0ee7-44e1-a760-c298d88c1a95\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4307145a-0ee7-44e1-a760-c298d88c1a95\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.716301002Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"region":"4307145a-0ee7-44e1-a760-c298d88c1a95","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4307145a-0ee7-44e1-a760-c298d88c1a95\" \"--data\" \"/data/regions/4307145a-0ee7-44e1-a760-c298d88c1a95\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.718893052Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"region":"4307145a-0ee7-44e1-a760-c298d88c1a95","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.828748328Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions","method":"POST","req_id":"34a007f8-73ae-4f7b-83ab-a580f397c14b","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":106,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.023217453Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"region":"4307145a-0ee7-44e1-a760-c298d88c1a95","component":"worker"}
{"msg":"region 4307145a-0ee7-44e1-a760-c298d88c1a95 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.023272263Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.023359864Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.023812228Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.033956773Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.053738008Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.056987118Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.057556223Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.057617144Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.057666564Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.057700735Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.06150371Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.06573763Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:39.262609257Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions","method":"POST","req_id":"c1497424-0948-4ff2-9992-44b3caef17e4","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.486203486Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"GET","req_id":"64ab3e45-ed31-40ab-b071-2b57f9d92365","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":690,"response_code":"200"}
{"msg":"path is \"/data/regions/4307145a-0ee7-44e1-a760-c298d88c1a95\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.489982381Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"dataset is oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.50058321Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"region 4307145a-0ee7-44e1-a760-c298d88c1a95 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.510304261Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.510461672Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.510533243Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"DELETE","req_id":"51f243e9-3945-40b6-93f1-81292ef6d434","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20575,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.513646882Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"GET","req_id":"99d5f76a-3760-4d76-8d0c-9679b239960b","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-4307145a-0ee7-44e1-a760-c298d88c1a95 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.513866134Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.542847084Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cab635fb-9210-4050-9597-9cfcbcba32c9/crucible/regions/4307145a-0ee7-44e1-a760-c298d88c1a95\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.552814977Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"region":"4307145a-0ee7-44e1-a760-c298d88c1a95","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.586907806Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"GET","req_id":"65751934-4192-49aa-aea8-ca77daf6e78b","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.700089312Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"GET","req_id":"08c2d682-a38a-4a43-84e5-6f435c2644b5","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"region 4307145a-0ee7-44e1-a760-c298d88c1a95 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.713287015Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.825525493Z","hostname":"oxz_crucible_8aaf2650-aaf7-4de6-8435-d480cf484556","pid":23243,"uri":"/crucible/0/regions/4307145a-0ee7-44e1-a760-c298d88c1a95","method":"GET","req_id":"ec744bed-3937-45f9-8bc2-62da220c67b8","remote_addr":"[fd00:1122:3344:101::a]:56006","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":31,"response_code":"200"}