/** * Waits for the given mapping type to exists on the master node. */ public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception { GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index); assertThat(mappings, notNullValue()); MappingMetaData mappingMetaData = mappings.get(type); assertThat(mappingMetaData, notNullValue()); Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap(); assertFalse(mappingSource.isEmpty()); assertTrue(mappingSource.containsKey("properties")); for (String fieldName : fieldNames) { Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties"); if (fieldName.indexOf('.') != -1) { fieldName = fieldName.replace(".", ".properties."); } assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); } }
@Override public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.type()); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry<String, Object> entry = iterator.next(); String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); if (fieldName.equals("type")) { builder.type(fieldNode.toString()); iterator.remove(); } else if (FIELDDATA.match(fieldName)) { // for bw compat only Map<String, String> fieldDataSettings = SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(fieldNode, "fielddata")); if (fieldDataSettings.containsKey("loading")) { builder.eagerGlobalOrdinals("eager_global_ordinals".equals(fieldDataSettings.get("loading"))); } iterator.remove(); } else if (fieldName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(fieldNode, "eager_global_ordinals")); iterator.remove(); } } return builder; }
public void testCompletionMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") .addMapping("my-type", createMappingSource("completion")) ); GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); assertThat(mappingMetaData, not(nullValue())); Map<String, Object> mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); assertThat(aField.size(), equalTo(6)); assertThat(aField.get("type").toString(), equalTo("completion")); assertThat(aField.get("fields"), notNullValue()); Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefreshPolicy(IMMEDIATE).get(); SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")).get(); assertThat(countResponse.getHits().getTotalHits(), equalTo(1L)); }
public void testIpMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") .addMapping("my-type", createMappingSource("ip")) ); GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); assertThat(mappingMetaData, not(nullValue())); Map<String, Object> mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); assertThat(aField.size(), equalTo(2)); assertThat(aField.get("type").toString(), equalTo("ip")); assertThat(aField.get("fields"), notNullValue()); Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "127.0.0.1").setRefreshPolicy(IMMEDIATE).get(); SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")).get(); assertThat(countResponse.getHits().getTotalHits(), equalTo(1L)); }
public SnapshotsSettings(RiverSettings settings) { super(); if (settings.settings().containsKey(RIVERNAME)) { @SuppressWarnings({ "unchecked" }) Map<String, Object> snapshotterSettings = (Map<String, Object>) settings.settings().get(RIVERNAME); this.repository = XContentMapValues.nodeStringValue(snapshotterSettings.get("repository"), "my_backup"); this.indices = XContentMapValues.nodeStringValue(snapshotterSettings.get("indices"), "_all"); this.includeGlobalState = XContentMapValues.nodeBooleanValue(snapshotterSettings.get("include_global_state"), false); this.frequency = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(snapshotterSettings.get("frequency"), "24h"), TimeValue.timeValueMinutes(60)); if (snapshotterSettings.get("purgeAfter") != null && snapshotterSettings.get("purgeAfter").toString().length() > 0) { this.purgeAfter = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(snapshotterSettings.get("purgeAfter"), "240h"), TimeValue.timeValueHours(240)); } else { this.purgeAfter = null; } this.setPurgeIndicesMustMatch(XContentMapValues.nodeBooleanValue(snapshotterSettings.get("purge_indices_must_match"), true)); } else { this.repository = "my_backup"; this.indices = "_all"; this.includeGlobalState = false; this.frequency = TimeValue.timeValueHours(24); this.purgeAfter = null; // no purging by default this.setPurgeIndicesMustMatch(true); } }
@SuppressWarnings({"unchecked", "rawtypes"}) @Override public Mapper.Builder parse(String name, Map<String, Object> mapping, ParserContext parserContext) { StandardnumberMapper.Builder builder = new Builder(name, service); Iterator<Map.Entry<String, Object>> iterator = mapping.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Object> entry = iterator.next(); String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); switch (fieldName) { case "standardnumbers" : builder.settingsBuilder.putArray("standardnumbers", XContentMapValues.nodeStringArrayValue(fieldNode)); iterator.remove(); break; default: break; } } return builder; }
private Feeder createFeeder(String riverType, String riverName, RiverSettings riverSettings) { JDBCFeeder feeder = null; try { Map<String, Object> spec = (Map<String, Object>) riverSettings.settings().get("jdbc"); Map<String, String> loadedSettings = new JsonSettingsLoader().load(jsonBuilder().map(spec).string()); Settings mySettings = settingsBuilder().put(loadedSettings).build(); String strategy = XContentMapValues.nodeStringValue(spec.get("strategy"), "simple"); RiverFlow riverFlow = RiverServiceLoader.findRiverFlow(strategy); logger.debug("found river flow class {} for strategy {}", riverFlow.getClass().getName(), strategy); feeder = riverFlow.getFeeder(); logger.debug("spec = {} settings = {}", spec, mySettings.getAsMap()); feeder.setName(riverName) .setType(riverType) .setSpec(spec).setSettings(mySettings); } catch (IOException e) { logger.error(e.getMessage(), e); } return feeder; }
@Override protected void createRiverContext(String riverType, String riverName, Map<String, Object> mySettings) throws IOException { super.createRiverContext(riverType, riverName, mySettings); // defaults for column strategy String columnCreatedAt = XContentMapValues.nodeStringValue(mySettings.get("created_at"), "created_at"); String columnUpdatedAt = XContentMapValues.nodeStringValue(mySettings.get("updated_at"), "updated_at"); String columnDeletedAt = XContentMapValues.nodeStringValue(mySettings.get("deleted_at"), null); boolean columnEscape = XContentMapValues.nodeBooleanValue(mySettings.get("column_escape"), true); TimeValue lastRunTimeStampOverlap = XContentMapValues.nodeTimeValue(mySettings.get("last_run_timestamp_overlap"), TimeValue.timeValueSeconds(0)); riverContext .columnCreatedAt(columnCreatedAt) .columnUpdatedAt(columnUpdatedAt) .columnDeletedAt(columnDeletedAt) .columnEscape(columnEscape) .setLastRunTimeStampOverlap(lastRunTimeStampOverlap); }
@Test public void testAttachments() throws Exception{ Map<String, Object> settings = settings("/river-imap-attachments.json"); final Properties props = new Properties(); final String user = XContentMapValues.nodeStringValue(settings.get("user"), null); final String password = XContentMapValues.nodeStringValue(settings.get("password"), null); for (final Map.Entry<String, Object> entry : settings.entrySet()) { if (entry != null && entry.getKey().startsWith("mail.")) { props.setProperty(entry.getKey(), String.valueOf(entry.getValue())); } } registerRiver("imap_river", "river-imap-attachments.json"); final Session session = Session.getInstance(props); final Store store = session.getStore(); store.connect(user, password); checkStoreForTestConnection(store); final Folder inbox = store.getFolder("INBOX"); inbox.open(Folder.READ_WRITE); final MimeMessage message = new MimeMessage(session); message.setFrom(new InternetAddress(EMAIL_TO)); message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS)); message.setSubject(EMAIL_SUBJECT + "::attachment test"); message.setSentDate(new Date()); BodyPart bp = new MimeBodyPart(); bp.setText("Text"); Multipart mp = new MimeMultipart(); mp.addBodyPart(bp); bp = new MimeBodyPart(); DataSource ds = new ByteArrayDataSource(this.getClass().getResourceAsStream("/httpclient-tutorial.pdf"), AttachmentMapperTest.APPLICATION_PDF); bp.setDataHandler(new DataHandler(ds)); bp.setFileName("httpclient-tutorial.pdf"); mp.addBodyPart(bp); message.setContent(mp); inbox.appendMessages(new Message[]{message}); IMAPUtils.close(inbox); IMAPUtils.close(store); //let the river index Thread.sleep(20*1000); esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet(); SearchResponse searchResponse = esSetup.client().prepareSearch("imapriverdata").setTypes("mail").execute().actionGet(); Assert.assertEquals(1, searchResponse.getHits().totalHits()); //BASE64 content httpclient-tutorial.pdf Assert.assertTrue(searchResponse.getHits().hits()[0].getSourceAsString().contains(AttachmentMapperTest.PDF_BASE64_DETECTION)); searchResponse = esSetup.client().prepareSearch("imapriverdata").addFields("*").setTypes("mail").setQuery(QueryBuilders.matchPhraseQuery("attachments.content.content", PDF_CONTENT_TO_SEARCH)).execute().actionGet(); Assert.assertEquals(1, searchResponse.getHits().totalHits()); Assert.assertEquals(1, searchResponse.getHits().hits()[0].field("attachments.content.content").getValues().size()); Assert.assertEquals("HttpClient Tutorial", searchResponse.getHits().hits()[0].field("attachments.content.title").getValue().toString()); Assert.assertEquals("application/pdf", searchResponse.getHits().hits()[0].field("attachments.content.content_type").getValue().toString()); Assert.assertTrue(searchResponse.getHits().hits()[0].field("attachments.content.content").getValue().toString().contains(PDF_CONTENT_TO_SEARCH)); }
private boolean isStarted(){ // Refresh index before querying it. client.admin().indices().prepareRefresh("_river").execute().actionGet(); GetResponse isStartedGetResponse = client.prepareGet("_river", riverName().name(), "_s3status").execute().actionGet(); try{ if (!isStartedGetResponse.isExists()){ XContentBuilder xb = jsonBuilder().startObject() .startObject("amazon-s3") .field("feedname", feedDefinition.getFeedname()) .field("status", "STARTED").endObject() .endObject(); client.prepareIndex("_river", riverName.name(), "_s3status").setSource(xb).execute(); return true; } else { String status = (String)XContentMapValues.extractValue("amazon-s3.status", isStartedGetResponse.getSourceAsMap()); if ("STOPPED".equals(status)){ return false; } } } catch (Exception e){ logger.warn("failed to get status for " + riverName().name() + ", throttling....", e); } return true; }
/** * Extract array from settings (array or ; delimited String) * @param settings Settings * @param path Path to settings definition * @return Array of settings */ @SuppressWarnings("unchecked") public static String[] buildArrayFromSettings(Map<String, Object> settings, String path){ String[] includes; // We manage comma separated format and arrays if (XContentMapValues.isArray(XContentMapValues.extractValue(path, settings))) { List<String> includesarray = (List<String>) XContentMapValues.extractValue(path, settings); int i = 0; includes = new String[includesarray.size()]; for (String include : includesarray) { includes[i++] = trimAllWhitespace(include); } } else { String includedef = (String) XContentMapValues.extractValue(path, settings); includes = Strings.commaDelimitedListToStringArray(trimAllWhitespace(includedef)); } String[] uniquelist = removeDuplicateStrings(includes); return uniquelist; }
/** * Extract array from settings (array or ; delimited String) * @param settings Settings * @param path Path to settings definition * @return Array of settings */ @SuppressWarnings("unchecked") public static String[] buildArrayFromSettings(Map<String, Object> settings, String path){ String[] includes; // We manage comma separated format and arrays if (XContentMapValues.isArray(XContentMapValues.extractValue(path, settings))) { List<String> includesarray = (List<String>) XContentMapValues.extractValue(path, settings); int i = 0; includes = new String[includesarray.size()]; for (String include : includesarray) { includes[i++] = Strings.trimAllWhitespace(include); } } else { String includedef = (String) XContentMapValues.extractValue(path, settings); includes = Strings.commaDelimitedListToStringArray(Strings.trimAllWhitespace(includedef)); } String[] uniquelist = Strings.removeDuplicateStrings(includes); return uniquelist; }
private boolean isStarted(){ // Refresh index before querying it. client.admin().indices().prepareRefresh("_river").execute().actionGet(); GetResponse isStartedGetResponse = client.prepareGet("_river", riverName().name(), "_drivestatus").execute().actionGet(); try{ if (!isStartedGetResponse.isExists()){ XContentBuilder xb = jsonBuilder().startObject() .startObject("google-drive") .field("feedname", feedDefinition.getFeedname()) .field("status", "STARTED").endObject() .endObject(); client.prepareIndex("_river", riverName.name(), "_drivestatus").setSource(xb).execute(); return true; } else { String status = (String)XContentMapValues.extractValue("google-drive.status", isStartedGetResponse.getSourceAsMap()); if ("STOPPED".equals(status)){ return false; } } } catch (Exception e){ logger.warn("failed to get status for " + riverName().name() + ", throttling....", e); } return true; }
private void parseConfig() { if (settings.settings().containsKey("wildfly")) { Map<String, Object> wildflySettings = (Map<String, Object>) settings.settings().get("wildfly"); this.username = XContentMapValues.nodeStringValue(wildflySettings.get("user"), null); this.password = XContentMapValues.nodeStringValue(wildflySettings.get("user"), null); this.host = XContentMapValues.nodeStringValue(wildflySettings.get("host"), null); this.port = XContentMapValues.nodeIntegerValue(wildflySettings.get("port"), 9999); this.scheduleSeconds = XContentMapValues.nodeIntegerValue(wildflySettings.get("schedule"), 1); } else { logger.error("invalid wildfly plugin configuration"); } if (settings.settings().containsKey("index")) { Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index"); indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name()); typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "status"); } else { logger.error("invalid wildfly plugin configuration"); } }
@SuppressWarnings("unchecked") private List<Attribute> nodeToAttributeList(Object obj, List<Attribute> defaultValue) { if (null != obj && XContentMapValues.isArray(obj)) { List<Attribute> res = new ArrayList<Attribute>(); for (Object o : (List<Object>) obj) { if (o instanceof String) { res.add(new Attribute(o.toString())); } else if (o instanceof Map) { Map<String, Object> oMap = (Map<String, Object>) o; if (oMap.containsKey("name")) { if (oMap.containsKey("transform")) { res.add(new Attribute(oMap.get("name").toString(), oMap.get("transform").toString())); } else { res.add(new Attribute(oMap.get("name").toString())); } } } } return res; } return defaultValue; }
private void loadDefaultsIfNecessary() { Map<String, Object> settingsDefault = loadDefaultSettingsMapFromFile(); indexFieldForRiverName = loadDefaultStringIfNecessary(indexFieldForRiverName, CONFIG_FIELDRIVERNAME, settingsDefault); indexFieldForSpaceKey = loadDefaultStringIfNecessary(indexFieldForSpaceKey, CONFIG_FIELDSPACEKEY, settingsDefault); indexFieldForRemoteDocumentId = loadDefaultStringIfNecessary(indexFieldForRemoteDocumentId, CONFIG_FIELDDOCUMENTID, settingsDefault); if (filtersConfig == null) filtersConfig = new HashMap<String, Map<String, String>>(); if (commentIndexingMode == null) { commentIndexingMode = CommentIndexingMode.parseConfiguration(XContentMapValues.nodeStringValue( settingsDefault.get(CONFIG_COMMENTMODE), null)); } }
@Override public Date extractDocumentUpdated(Map<String, Object> document) { Object val = XContentMapValues.extractValue(remoteDataFieldForUpdated, document); if (val == null) return null; if (!Utils.isSimpleValue(val)) throw new SettingsException("Remote data field '" + remoteDataFieldForUpdated + "' must provide simple value, but value is " + val); if (val instanceof Date) return (Date) val; try { return DateTimeUtils.parseDate( val.toString(), remoteDataFieldForUpdatedFormat ); } catch (IllegalArgumentException e1) { throw new SettingsException("Remote data field '" + remoteDataFieldForUpdated + "' is not reecognized: " + val); } }
@Override public boolean extractDocumentDeleted(Map<String, Object> document) { if (document == null || remoteDataFieldForDeleted == null) return false; Object val = XContentMapValues.extractValue(remoteDataFieldForDeleted, document); if (val == null) return false; if (!Utils.isSimpleValue(val)) throw new SettingsException("Remote data field '" + remoteDataFieldForDeleted + "' must provide simple value, but value is " + val); String v = null; if (val instanceof String) { v = (String) val; } else { v = val.toString(); } return v.equals(remoteDataValueForDeleted); }
@Override public HashMap<String, String> loadKey(String username) { logger.info("loading password for username {}", username); HashMap<String, String> ret = new HashMap<>(); String riverIndexName = getRiverIndexName(); refreshSearchIndex(riverIndexName); GetResponse resp = client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet(); if (resp.isExists()) { if (logger.isDebugEnabled()) { logger.debug("Password document: {}", resp.getSourceAsString()); } Map<String, Object> newset = resp.getSource(); Set<String> keys = newset.keySet(); for(String s : keys){ logger.info("Added key {} with a value of {}",s,XContentMapValues.nodeStringValue(newset.get(s), null)); ret.put(s,XContentMapValues.nodeStringValue(newset.get(s), null)); } } if(ret.isEmpty()){ return null; } return ret; }
/** * Get url from configuration and validate it for format, and for presence. * * @param config to get URL from * @param cfgProperyName name of config property with URL * @param mandatory if URL is mandatory so validation is performed * @return url * @throws SettingsException in case of validation error */ protected static String getUrlFromConfig(Map<String, Object> config, String cfgProperyName, boolean mandatory) throws SettingsException { String url = null; if (config != null) url = Utils.trimToNull(XContentMapValues.nodeStringValue(config.get(cfgProperyName), null)); if (mandatory && url == null) { throw new SettingsException("remote/" + cfgProperyName + " element of configuration structure not found or empty"); } if (url != null) { try { new URL(url); } catch (MalformedURLException e) { throw new SettingsException("Parameter remote/" + cfgProperyName + " is malformed URL " + e.getMessage()); } } return url; }
/** * Returns a filter function that expects the source map as an input and returns * the filtered map. */ public Function<Map<String, ?>, Map<String, Object>> getFilter() { if (filter == null) { filter = XContentMapValues.filter(includes, excludes); } return filter; }
private List<Object> fetch(TermsLookup termsLookup, Client client) { List<Object> terms = new ArrayList<>(); GetRequest getRequest = new GetRequest(termsLookup.index(), termsLookup.type(), termsLookup.id()) .preference("_local").routing(termsLookup.routing()); final GetResponse getResponse = client.get(getRequest).actionGet(); if (getResponse.isSourceEmpty() == false) { // extract terms only if the doc source exists List<Object> extractedValues = XContentMapValues.extractRawValues(termsLookup.path(), getResponse.getSourceAsMap()); terms.addAll(extractedValues); } return terms; }
private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes, Settings indexSettings) { super(NAME, Defaults.FIELD_TYPE.clone(), Defaults.FIELD_TYPE, indexSettings); // Only stored. this.enabled = enabled; this.includes = includes; this.excludes = excludes; final boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0); this.filter = enabled && filtered && fieldType().stored() ? XContentMapValues.filter(includes, excludes) : null; this.complete = enabled && includes == null && excludes == null; }
@Override public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name); parseField(builder, name, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry<String, Object> entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); if (propName.equals("null_value")) { if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); } builder.nullValue(propNode.toString()); iterator.remove(); } else if (propName.equals("ignore_above")) { builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1)); iterator.remove(); } else if (propName.equals("norms")) { builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, "norms") == false); iterator.remove(); } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals")); iterator.remove(); } else if (propName.equals("normalizer")) { if (propNode != null) { NamedAnalyzer normalizer = parserContext.getIndexAnalyzers().getNormalizer(propNode.toString()); if (normalizer == null) { throw new MapperParsingException("normalizer [" + propNode.toString() + "] not found for field [" + name + "]"); } builder.normalizer(normalizer); } iterator.remove(); } } return builder; }
@Override public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName); builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer()); builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer()); builder.fieldType().setSearchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()); parseTextField(builder, fieldName, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry<String, Object> entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); if (propName.equals("position_increment_gap")) { int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1); builder.positionIncrementGap(newPositionIncrementGap); iterator.remove(); } else if (propName.equals("fielddata")) { builder.fielddata(XContentMapValues.nodeBooleanValue(propNode, "fielddata")); iterator.remove(); } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals")); iterator.remove(); } else if (propName.equals("fielddata_frequency_filter")) { Map<?,?> frequencyFilter = (Map<?, ?>) propNode; double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0); double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE); int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0); builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); } } return builder; }
public static boolean nodeBooleanValue(String fieldName, String propertyName, Object node, Mapper.TypeParser.ParserContext parserContext) { if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { return XContentMapValues.nodeBooleanValue(node, fieldName + "." + propertyName); } else { return nodeBooleanValueLenient(fieldName, propertyName, node); } }
public void testGeoPointMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") .addMapping("my-type", createMappingSource("geo_point")) ); GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); assertThat(mappingMetaData, not(nullValue())); Map<String, Object> mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); logger.info("Keys: {}", aField.keySet()); assertThat(aField.size(), equalTo(2)); assertThat(aField.get("type").toString(), equalTo("geo_point")); assertThat(aField.get("fields"), notNullValue()); Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); GeoPoint point = new GeoPoint(51, 19); client().prepareIndex("my-index", "my-type", "1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get(); SearchResponse countResponse = client().prepareSearch("my-index").setSize(0) .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))) .get(); assertThat(countResponse.getHits().getTotalHits(), equalTo(1L)); countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())).get(); assertThat(countResponse.getHits().getTotalHits(), equalTo(1L)); }
public void testTokenCountMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") .addMapping("my-type", XContentFactory.jsonBuilder().startObject().startObject("my-type") .startObject("properties") .startObject("a") .field("type", "token_count") .field("analyzer", "simple") .startObject("fields") .startObject("b") .field("type", "keyword") .endObject() .endObject() .endObject() .endObject() .endObject().endObject()) ); GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); assertThat(mappingMetaData, not(nullValue())); Map<String, Object> mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); assertThat(aField.size(), equalTo(3)); assertThat(aField.get("type").toString(), equalTo("token_count")); assertThat(aField.get("fields"), notNullValue()); Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); assertThat(bField.size(), equalTo(1)); assertThat(bField.get("type").toString(), equalTo("keyword")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "my tokens").setRefreshPolicy(IMMEDIATE).get(); SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "my tokens")).get(); assertThat(countResponse.getHits().getTotalHits(), equalTo(1L)); }
@Nullable @Override public Object referenceValue(Reference reference) { if (updatedColumnValues == null) { return super.referenceValue(reference); } Object value = updatedColumnValues.get(reference.ident().columnIdent().fqn()); if (value == null && !reference.ident().isColumn()) { value = XContentMapValues.extractValue(reference.ident().columnIdent().fqn(), updatedColumnValues); } return reference.valueType().value(value); }
@Override public Function<GetResult, Object> build(final Reference reference, SymbolToFieldExtractor.Context context) { return new Function<GetResult, Object>() { @Override public Object apply(GetResult getResult) { if (getResult == null) { return null; } return reference.valueType().value(XContentMapValues.extractValue( reference.info().ident().columnIdent().fqn(), getResult.sourceAsMap())); } }; }
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { Builder builder = geoShapeField(name); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry<String, Object> entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); if (Names.TREE.equals(fieldName)) { builder.fieldType().setTree(fieldNode.toString()); iterator.remove(); } else if (Names.TREE_LEVELS.equals(fieldName)) { builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); iterator.remove(); } else if (Names.TREE_PRESISION.equals(fieldName)) { builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); iterator.remove(); } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); iterator.remove(); } else if (Names.ORIENTATION.equals(fieldName)) { builder.fieldType().setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString())); iterator.remove(); } else if (Names.STRATEGY.equals(fieldName)) { builder.fieldType().setStrategyName(fieldNode.toString()); iterator.remove(); } else if (Names.COERCE.equals(fieldName)) { builder.coerce(nodeBooleanValue(fieldNode)); iterator.remove(); } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName) && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) { builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode)); iterator.remove(); } } return builder; }
@Override public ExecutableScript newScript(@Nullable Map<String, Object> params) { String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); String hashValue = params == null ? null : XContentMapValues.nodeStringValue(params.get("hash"), null); if (fieldName == null) { throw new ScriptException("Missing the field parameter"); } return new HammingDistanceScript(fieldName, hashValue); }
public ExecutableScript newScript(@Nullable Map<String, Object> params) { String pattern = params == null ? "" : XContentMapValues.nodeStringValue(params.get("pattern"), ""); String fieldName = params == null ? "" : XContentMapValues.nodeStringValue(params.get("fieldName"), ""); String groupkeys = params == null ? "" : XContentMapValues.nodeStringValue(params.get("groupkeys"), ""); String isHashMap = params == null ? "" : XContentMapValues.nodeStringValue(params.get("isHashMap"), ""); if (StringUtils.isBlank(fieldName)) { throw new ScriptException("Missing field parameter"); } if (StringUtils.isBlank(pattern)) { throw new ScriptException("Missing field parameter"); } List<String> groupkeyList = new ArrayList<>(); if (StringUtils.isNotBlank(groupkeys)) { groupkeyList = Arrays.asList(groupkeys.split(",")); } Boolean isHashMapBoolean; if (StringUtils.isBlank(isHashMap) || "false".equals(isHashMap.toLowerCase())) { isHashMapBoolean = false; } else { isHashMapBoolean = true; } return new GrokNativeScript(pattern, fieldName, groupkeyList, isHashMapBoolean); }
@Override public ExecutableScript newScript(@Nullable Map<String, Object> params) { if (params == null || params.containsKey("spec") == false) { throw new IllegalArgumentException("the spec parameter is required"); } Map<String, Object> spec = XContentMapValues.nodeMapValue(params.get("spec"), "spec"); // TODO: Add caching mechanism VectorRangesToVector features = new VectorRangesToVectorJSON(spec); return new VectorizerScript(features); }
/** * Waits for the given mapping type to exists on the master node. */ public void waitForMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception { assertBusy(new Callable() { @Override public Object call() throws Exception { final GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); final ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index); assertThat(mappings, notNullValue()); final MappingMetaData mappingMetaData = mappings.get(type); assertThat(mappingMetaData, notNullValue()); final Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap(); assertFalse(mappingSource.isEmpty()); assertTrue(mappingSource.containsKey("properties")); for (String fieldName : fieldNames) { final Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties"); if (fieldName.indexOf('.') != -1) { fieldName = fieldName.replace(".", ".properties."); } assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); } return null; } }); }
@SuppressWarnings({ "unchecked" }) private String readConfig(final String config, final String defaultValue) { if (settings.settings().containsKey("bigquery")) { Map<String, Object> bqSettings = (Map<String, Object>) settings.settings().get("bigquery"); return XContentMapValues.nodeStringValue(bqSettings.get(config), defaultValue); } return defaultValue; }
@SuppressWarnings({"unchecked"}) @Inject public GitHubRiver(RiverName riverName, RiverSettings settings, Client client) { super(riverName, settings); this.client = client; if (!settings.settings().containsKey("github")) { throw new IllegalArgumentException("Need river settings - owner and repository."); } // get settings Map<String, Object> githubSettings = (Map<String, Object>) settings.settings().get("github"); owner = XContentMapValues.nodeStringValue(githubSettings.get("owner"), null); repository = XContentMapValues.nodeStringValue(githubSettings.get("repository"), null); index = String.format("%s&%s", owner, repository); userRequestedInterval = XContentMapValues.nodeIntegerValue(githubSettings.get("interval"), 60); // auth (optional) username = null; password = null; if (githubSettings.containsKey("authentication")) { Map<String, Object> auth = (Map<String, Object>) githubSettings.get("authentication"); username = XContentMapValues.nodeStringValue(auth.get("username"), null); password = XContentMapValues.nodeStringValue(auth.get("password"), null); } // endpoint (optional - default to github.com) endpoint = XContentMapValues.nodeStringValue(githubSettings.get("endpoint"), "https://api.github.com"); logger.info("Created GitHub river."); }
/** * This method is called for every search on every shard. * * @param params list of script parameters passed with the query * @return new native script */ @Override public ExecutableScript newScript(@Nullable Map<String, Object> params) { if (params == null) { throw new ScriptException("Missing script parameters"); } // stoic specific concept. if the name of the index for the hit is "b-something" // and no index is defined, we generate the lookup index by replacing the hit's index "b-" by "u-$tenant-" String tenant = XContentMapValues.nodeStringValue(params.get("tenant"), null); String index = XContentMapValues.nodeStringValue(params.get("index"), null); String type = XContentMapValues.nodeStringValue(params.get("type"), null); String id = XContentMapValues.nodeStringValue(params.get("id"), null); String sourceIndex = XContentMapValues.nodeStringValue(params.get("source_index"), null); String sourceType = XContentMapValues.nodeStringValue(params.get("source_type"), null); String sourceId = XContentMapValues.nodeStringValue(params.get("source_id"), null); String targetSource = XContentMapValues.nodeStringValue(params.get("target_source"), null); String targetSourceIncludes = XContentMapValues.nodeStringValue(params.get("target_source_includes"), null); String targetSourceExcludes = XContentMapValues.nodeStringValue(params.get("target_source_excludes"), null); String targetSourceLwc = XContentMapValues.nodeStringValue(params.get("target_source_lwc"), null); String missing = XContentMapValues.nodeStringValue(params.get("missing"), null); String cacheFieldSave = XContentMapValues.nodeStringValue(params.get("cache_field_save"), null); // write a cache field on the document. String cachedFieldRead = XContentMapValues.nodeStringValue(params.get("cached_field"), null); return new JoinerScript(node.client(), logger, cache, tenant, index, type, id, sourceIndex, sourceType, sourceId, targetSource, targetSourceIncludes, targetSourceExcludes, targetSourceLwc, missing, cacheFieldSave, cachedFieldRead); }
@SuppressWarnings({"unchecked"}) public static List<SQLCommand> parse(Map<String, Object> settings) { List<SQLCommand> sql = new LinkedList<SQLCommand>(); if (!XContentMapValues.isArray(settings.get("sql"))) { settings.put("sql", Arrays.asList(settings.get("sql"))); } List<Object> list = (List<Object>) settings.get("sql"); for (Object entry : list) { SQLCommand command = new SQLCommand(); try { if (entry instanceof Map) { Map<String, Object> m = (Map<String, Object>) entry; if (m.containsKey("statement")) { command.setSQL((String) m.get("statement")); } if (m.containsKey("parameter")) { command.setParameters(XContentMapValues.extractRawValues("parameter", m)); } if (m.containsKey("callable")) { command.setCallable(XContentMapValues.nodeBooleanValue(m.get("callable"))); } if (m.containsKey("register")) { command.setRegister(XContentMapValues.nodeMapValue(m.get("register"), null)); } } else if (entry instanceof String) { command.setSQL((String) entry); } sql.add(command); } catch (IOException e) { throw new IllegalArgumentException("SQL command not found", e); } } return sql; }
private void setContextSettings(String riverResource) throws IOException { RiverSettings riverSettings = riverSettings(riverResource); Map<String, Object> settings = (Map<String, Object>) riverSettings.settings().get("jdbc"); riverSettings.settings().put(ColumnRiverFlow.LAST_RUN_TIME, LAST_RUN_TIME); riverSettings.settings().put(ColumnRiverFlow.CURRENT_RUN_STARTED_TIME, new TimeValue(new Date().getTime())); context.setRiverSettings(riverSettings.settings()); context.setStatements(SQLCommand.parse(settings)); context.columnCreatedAt(XContentMapValues.nodeStringValue(settings.get("column_created_at"), null)); context.columnUpdatedAt(XContentMapValues.nodeStringValue(settings.get("column_updated_at"), null)); context.columnDeletedAt(XContentMapValues.nodeStringValue(settings.get("column_deleted_at"), null)); context.columnEscape(true); context.setLastRunTimeStampOverlap(getLastRunTimestampOverlap(riverSettings)); }