[ Oct 14 03:57:31 Enabled. ]
[ Oct 14 03:57:31 Rereading configuration. ]
[ Oct 14 03:57:32 Rereading configuration. ]
[ Oct 14 03:57:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:57:36.741504747Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:57:36.742875522Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:57:36.742887172Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:57:36.757927032Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:57:36.972767706Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.309875365Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47398"}
{"msg":"region 03a183a5-5c70-475a-b4ba-5166e79b5994 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.333001959Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.337572534Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions","method":"POST","req_id":"67a1d9e3-6d23-40c2-84ec-3042e07bf2a1","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":9328,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.337599145Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.347517486Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.347553397Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"03a183a5-5c70-475a-b4ba-5166e79b5994\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/03a183a5-5c70-475a-b4ba-5166e79b5994\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.415622897Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"region":"03a183a5-5c70-475a-b4ba-5166e79b5994","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"03a183a5-5c70-475a-b4ba-5166e79b5994\" \"--data\" \"/data/regions/03a183a5-5c70-475a-b4ba-5166e79b5994\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.419460474Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"region":"03a183a5-5c70-475a-b4ba-5166e79b5994","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.597315147Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions","method":"POST","req_id":"925d0324-4c7d-4267-bed5-f4786249dddc","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.681425452Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"region":"03a183a5-5c70-475a-b4ba-5166e79b5994","component":"worker"}
{"msg":"region 03a183a5-5c70-475a-b4ba-5166e79b5994 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.681477663Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.681576085Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.681977606Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.686857009Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.702692529Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.70630441Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.710996929Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.71106259Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.711116822Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.711166073Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.715052171Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:34.71937128Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.113987531Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions","method":"POST","req_id":"6502b4e7-992d-4727-9c43-92563597a2c5","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.723356897Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","method":"GET","req_id":"7c942a27-6060-4576-a99b-e8207b13e8bb","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"path is \"/data/regions/03a183a5-5c70-475a-b4ba-5166e79b5994\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.726138007Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"dataset is oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.737758641Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"region 03a183a5-5c70-475a-b4ba-5166e79b5994 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.746600434Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.746800219Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.746900052Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","method":"DELETE","req_id":"d0bacfc4-4f24-4fbd-8d9a-395137151e99","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20785,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-03a183a5-5c70-475a-b4ba-5166e79b5994 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.748029581Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.749896848Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","method":"GET","req_id":"06ec7de5-1a94-4cd8-8980-2abd9e1b881a","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.753755985Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_19134185-b184-4780-a06b-c6cab90733ad/crucible/regions/03a183a5-5c70-475a-b4ba-5166e79b5994\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.765345938Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"region":"03a183a5-5c70-475a-b4ba-5166e79b5994","component":"worker"}
{"msg":"region 03a183a5-5c70-475a-b4ba-5166e79b5994 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.828313179Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T03:58:35.891197708Z","hostname":"oxz_crucible_bdb4d30c-8e87-47a6-b88a-10571f1b24af","pid":21826,"uri":"/crucible/0/regions/03a183a5-5c70-475a-b4ba-5166e79b5994","method":"GET","req_id":"5dd786bf-ba99-4e3a-8e36-99f5299a3044","remote_addr":"[fd00:1122:3344:101::a]:47398","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":82,"response_code":"200"}