[ Feb 23 14:12:32 Enabled. ]
[ Feb 23 14:12:32 Rereading configuration. ]
[ Feb 23 14:12:34 Rereading configuration. ]
[ Feb 23 14:12:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.677113141Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.680352014Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.680361314Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.695691232Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.802999062Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.59735888Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:34495"}
{"msg":"region 4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.620253447Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.623704327Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions","method":"POST","req_id":"f8ed491c-41f0-423d-a7fc-8addac850897","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":12496,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.623725698Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.634445328Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.63449843Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.707116858Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\" \"--data\" \"/data/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.70907236Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.833016019Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","component":"worker"}
{"msg":"region 4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.833085861Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.833305298Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.833807104Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.837821952Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.849879255Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.85443359Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.855185394Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.85725713Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.857326842Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.857369833Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.867045121Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.872527435Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.938871094Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions","method":"POST","req_id":"c7214352-a594-409b-b745-410b6b87adb0","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.62565106Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","method":"GET","req_id":"cf6f60cf-6053-426c-ae7c-eaa281b3a4db","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.628687296Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"dataset is oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.640015996Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"region 4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.651625435Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.651819661Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.653063981Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.653750163Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","method":"DELETE","req_id":"80bc7b96-32d7-4d3c-8439-12c86a4133a4","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25088,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.657205773Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","method":"GET","req_id":"50cb3fe7-1a4e-4c5b-966d-b95299066a67","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.657959916Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.669161753Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","component":"worker"}
{"msg":"region 4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.752440179Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.782138553Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions/4b9b9ff3-65c3-4aa2-ab29-49d6eeb58b40","method":"GET","req_id":"e02396aa-265e-4433-ac30-57522bc549ff","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region 1f94d1e3-1581-4324-820c-87b31edc5420 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.418625224Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.420873115Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.420896986Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions","method":"POST","req_id":"e5fde3b8-6288-4ce0-88ce-a5b3818f18c1","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2313,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.430403938Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0b440700-a21e-4a8d-ae62-cbd6ce6b4459/crucible/regions/1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.430526762Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1f94d1e3-1581-4324-820c-87b31edc5420\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1f94d1e3-1581-4324-820c-87b31edc5420\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.502308313Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"1f94d1e3-1581-4324-820c-87b31edc5420","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1f94d1e3-1581-4324-820c-87b31edc5420\" \"--data\" \"/data/regions/1f94d1e3-1581-4324-820c-87b31edc5420\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.504155882Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"1f94d1e3-1581-4324-820c-87b31edc5420","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.574535789Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"region":"1f94d1e3-1581-4324-820c-87b31edc5420","component":"worker"}
{"msg":"region 1f94d1e3-1581-4324-820c-87b31edc5420 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.574601571Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.574785057Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.576165081Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.579766745Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.587187111Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.590406483Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1f94d1e3-1581-4324-820c-87b31edc5420","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.590680392Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.590739934Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.590787425Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.590832597Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.595096482Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.600211055Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.640592278Z","hostname":"oxz_crucible_c680bcf2-d60d-41a2-bfee-dd120429d312","pid":24069,"uri":"/crucible/0/regions","method":"POST","req_id":"c185e66a-b29b-4074-af18-cf1714ac429b","remote_addr":"[fd00:1122:3344:101::b]:34495","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}