[ Jul 30 01:30:26 Enabled. ]
[ Jul 30 01:30:26 Rereading configuration. ]
[ Jul 30 01:30:27 Rereading configuration. ]
[ Jul 30 01:30:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.699028215Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.700276701Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.700285961Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.717264384Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.784623867Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.462707378Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61778"}
{"msg":"region c358610c-c8b4-4495-a3ee-500720417559 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.483365157Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.488141256Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.488209477Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"c6f91de7-62ce-4064-a7fc-b5848330bb3f","remote_addr":"[fd00:1122:3344:101::a]:61778","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":13588,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.501460701Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.501499151Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c358610c-c8b4-4495-a3ee-500720417559\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c358610c-c8b4-4495-a3ee-500720417559\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.57631357Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"c358610c-c8b4-4495-a3ee-500720417559","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c358610c-c8b4-4495-a3ee-500720417559\" \"--data\" \"/data/regions/c358610c-c8b4-4495-a3ee-500720417559\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.578495092Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"c358610c-c8b4-4495-a3ee-500720417559","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.743347754Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"c358610c-c8b4-4495-a3ee-500720417559","component":"worker"}
{"msg":"region c358610c-c8b4-4495-a3ee-500720417559 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.743412785Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.743530926Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.743992351Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.748173403Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.762789432Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.765359858Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.773861284Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.775895335Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.775959075Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.776008686Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.779224558Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.783631973Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.864354992Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"e42c64d1-b5cd-4373-a5f4-e2d7ee34c2f4","remote_addr":"[fd00:1122:3344:101::a]:61778","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":733,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.812276002Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33597"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.812387514Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/c358610c-c8b4-4495-a3ee-500720417559","method":"GET","req_id":"7978a0a8-bc62-49c2-9109-7d8c44a2a441","remote_addr":"[fd00:1122:3344:101::a]:33597","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/c358610c-c8b4-4495-a3ee-500720417559\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.815428825Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"dataset is oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.824636688Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"snapshot is oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559@f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.833941122Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559\", \"f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.833968573Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.844829523Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/c358610c-c8b4-4495-a3ee-500720417559/snapshots/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","method":"GET","req_id":"825a5b50-d05d-409b-828d-951386b18ff3","remote_addr":"[fd00:1122:3344:101::a]:33597","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":32231,"response_code":"200"}
{"msg":"path is \"/data/regions/c358610c-c8b4-4495-a3ee-500720417559\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.845449079Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"dataset is oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.853680578Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"snapshot is oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559@f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.862113018Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/c358610c-c8b4-4495-a3ee-500720417559\", \"f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.862138578Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot c358610c-c8b4-4495-a3ee-500720417559-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.870648754Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions for region c358610c-c8b4-4495-a3ee-500720417559 running snapshot f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.871278371Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.871307071Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/c358610c-c8b4-4495-a3ee-500720417559/snapshots/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b/run","method":"POST","req_id":"85b1d79c-9826-4249-ad47-17fed4907ec2","remote_addr":"[fd00:1122:3344:101::a]:33597","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26341,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.87716796Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877259331Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877344912Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877353012Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877395683Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877446063Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877547274Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877613245Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877662015Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.877711736Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.885175152Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-c358610c-c8b4-4495-a3ee-500720417559-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.885642586Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c358610c-c8b4-4495-a3ee-500720417559-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.891724908Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c358610c-c8b4-4495-a3ee-500720417559-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.902879341Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c358610c-c8b4-4495-a3ee-500720417559/.zfs/snapshot/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.903174044Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.903228155Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.903272615Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.903311185Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.903359786Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.907112404Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.911805292Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"region c358610c-c8b4-4495-a3ee-500720417559 running snapshot f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.911822242Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.800929316Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62062"}
{"msg":"region 13190d17-ff58-4cdc-878d-9a10c2eb1a85 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.801297619Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.801471371Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.801489761Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"1305ef86-ea17-411d-82c9-c0eb56d4f7d4","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":485,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.812133479Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.81215975Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"13190d17-ff58-4cdc-878d-9a10c2eb1a85\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.880044998Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"13190d17-ff58-4cdc-878d-9a10c2eb1a85","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"13190d17-ff58-4cdc-878d-9a10c2eb1a85\" \"--data\" \"/data/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.880107289Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"13190d17-ff58-4cdc-878d-9a10c2eb1a85","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.94337227Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"13190d17-ff58-4cdc-878d-9a10c2eb1a85","component":"worker"}
{"msg":"region 13190d17-ff58-4cdc-878d-9a10c2eb1a85 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.943436141Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.943528402Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.943961486Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.946734134Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.953140069Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956408132Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956680295Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956737226Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956780736Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956825617Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.961581515Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.96607345Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.966192302Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.966284383Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.966290723Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.966330523Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.966373964Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.970466605Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.970524086Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.970573936Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.970628887Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.973674328Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.975688888Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:10.14843208Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"47419b42-2037-4d24-a54f-70526b8decec","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.880244127Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","method":"GET","req_id":"488413b0-7a14-41e1-a3b3-35a84d3f7a82","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"path is \"/data/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.884207708Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"dataset is oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.894593073Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"region 13190d17-ff58-4cdc-878d-9a10c2eb1a85 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.903692045Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.903864717Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.903917978Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","method":"DELETE","req_id":"633701d9-3e7f-4a1d-93f4-73f0b17ff085","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19736,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-13190d17-ff58-4cdc-878d-9a10c2eb1a85 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.904948478Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.907014159Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","method":"GET","req_id":"4cb495f7-fe51-4f42-b885-81ee71783016","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":729,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909278072Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909370703Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909453024Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909463314Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909508874Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909557085Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c358610c-c8b4-4495-a3ee-500720417559","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909883118Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.909960049Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.910015509Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.91006325Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.913345913Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.914694387Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0cb06715-9f13-41d3-ac8d-a156c2f253a3/crucible/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.924778909Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"region":"13190d17-ff58-4cdc-878d-9a10c2eb1a85","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.972012448Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","method":"GET","req_id":"ec1a96d6-d235-4eff-af96-723a497c2afc","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"region 13190d17-ff58-4cdc-878d-9a10c2eb1a85 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:33.975246421Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.050637146Z","hostname":"oxz_crucible_7fc3f873-b2b7-44ac-82be-d3338f22ec5d","pid":25546,"uri":"/crucible/0/regions/13190d17-ff58-4cdc-878d-9a10c2eb1a85","method":"GET","req_id":"48cde344-024d-4865-9c28-77c328e3ce66","remote_addr":"[fd00:1122:3344:101::a]:62062","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20,"response_code":"200"}