[ Nov 14 00:18:35 Enabled. ]
[ Nov 14 00:18:35 Rereading configuration. ]
[ Nov 14 00:18:40 Rereading configuration. ]
[ Nov 14 00:18:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:18:46.675527712Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:18:46.678434283Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:18:46.678447264Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:18:46.694702162Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:18:46.788068996Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.575072131Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52508"}
{"msg":"region 181a0fb3-9b28-4d00-9212-ad78d66d0ec2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.618594243Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.626237771Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"uri":"/crucible/0/regions","method":"POST","req_id":"5f2f362c-78e6-4b3f-bc33-a99f1696f159","remote_addr":"[fd00:1122:3344:101::a]:52508","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22034,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.626265931Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.639452627Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.639492978Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"181a0fb3-9b28-4d00-9212-ad78d66d0ec2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.689170686Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"181a0fb3-9b28-4d00-9212-ad78d66d0ec2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"181a0fb3-9b28-4d00-9212-ad78d66d0ec2\" \"--data\" \"/data/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.693752771Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"181a0fb3-9b28-4d00-9212-ad78d66d0ec2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:47.923627366Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"uri":"/crucible/0/regions","method":"POST","req_id":"a9abafaa-5bd3-4b08-8776-a9737f476176","remote_addr":"[fd00:1122:3344:101::a]:52508","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.034848811Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"181a0fb3-9b28-4d00-9212-ad78d66d0ec2","component":"worker"}
{"msg":"region 181a0fb3-9b28-4d00-9212-ad78d66d0ec2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.034899582Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.035049404Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.035672703Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.041475894Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.05683251Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.059833193Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.060580793Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.063042638Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.063094108Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.063130219Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.070527273Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.074100733Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:48.193749997Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"uri":"/crucible/0/regions","method":"POST","req_id":"eed1392e-df70-4c35-83ed-8ed380f6fbb3","remote_addr":"[fd00:1122:3344:101::a]:52508","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"region ed7362e3-4d5c-445e-84b0-f250003977c9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.329779671Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.329973124Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.329989685Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"uri":"/crucible/0/regions","method":"POST","req_id":"96966bb9-f6d8-4504-bf60-b1f86bf6f541","remote_addr":"[fd00:1122:3344:101::a]:52508","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":255,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible/regions/ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.340722456Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c2753ec7-decb-4fc2-b85a-bb303a9a0272/crucible/regions/ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.340764526Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ed7362e3-4d5c-445e-84b0-f250003977c9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ed7362e3-4d5c-445e-84b0-f250003977c9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.394393841Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"ed7362e3-4d5c-445e-84b0-f250003977c9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ed7362e3-4d5c-445e-84b0-f250003977c9\" \"--data\" \"/data/regions/ed7362e3-4d5c-445e-84b0-f250003977c9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.394447592Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"ed7362e3-4d5c-445e-84b0-f250003977c9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.468846479Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"region":"ed7362e3-4d5c-445e-84b0-f250003977c9","component":"worker"}
{"msg":"region ed7362e3-4d5c-445e-84b0-f250003977c9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.468884669Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.46897148Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470147817Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470236358Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470315969Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470322689Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.47036816Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470430731Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/181a0fb3-9b28-4d00-9212-ad78d66d0ec2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470530792Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470597973Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470647874Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.470710215Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.475570953Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.475989069Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.479372097Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.489117484Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.49242154Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed7362e3-4d5c-445e-84b0-f250003977c9","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.492684944Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.492749645Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.492793925Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.492844336Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.497270388Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.502164267Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-14T00:29:54.620067327Z","hostname":"oxz_crucible_ff5ba4ab-19ec-4089-86e3-65a9903e98eb","pid":25090,"uri":"/crucible/0/regions","method":"POST","req_id":"0ad1963a-e73f-4f4d-a7d6-54e7377a3169","remote_addr":"[fd00:1122:3344:101::a]:52508","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}