Remove duplicate values from JS array [duplicate]

Asked 2023-09-20 20:33:20 View 690,206

I have a very simple JavaScript array that may or may not contain duplicates.

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

I need to remove the duplicates and put the unique values in a new array.

I could point to all the code that I've tried but I think it's useless because they don't work. I accept jQuery solutions too.

Similar question:

  • _.uniq(peoplenames) solves this lodash.com/docs#uniq - anyone
  • @ConnorLeech its easy with lodash but not optimized way - anyone
  • The simplest approach (in my opinion) is to use the Set object which lets you store unique values of any type. In other words, Set will automatically remove duplicates for us. const names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"]; let unique = [...new Set(names)]; console.log(unique); // 'Mike', 'Matt', 'Nancy', 'Adam', 'Jenny', 'Carl' - anyone
  • There are too many Mikes in the world - why not remove them? Nancy got owned on this. - anyone
  • in my solution, I sort data before filtering : ` const result = data.sort().filter((v, idx, t) => idx==0 || v != t[idx-1]); - anyone

Answers

TL;DR

Using the Set constructor and the spread syntax:

uniq = [...new Set(array)];

( Note that var uniq will be an array... new Set() turns it into a set, but [... ] turns it back into an array again )


"Smart" but naïve way

uniqueArray = a.filter(function(item, pos) {
    return a.indexOf(item) == pos;
})

Basically, we iterate over the array and, for each element, check if the first position of this element in the array is equal to the current position. Obviously, these two positions are different for duplicate elements.

Using the 3rd ("this array") parameter of the filter callback we can avoid a closure of the array variable:

uniqueArray = a.filter(function(item, pos, self) {
    return self.indexOf(item) == pos;
})

Although concise, this algorithm is not particularly efficient for large arrays (quadratic time).

Hashtables to the rescue

function uniq(a) {
    var seen = {};
    return a.filter(function(item) {
        return seen.hasOwnProperty(item) ? false : (seen[item] = true);
    });
}

This is how it's usually done. The idea is to place each element in a hashtable and then check for its presence instantly. This gives us linear time, but has at least two drawbacks:

  • since hash keys can only be strings or symbols in JavaScript, this code doesn't distinguish numbers and "numeric strings". That is, uniq([1,"1"]) will return just [1]
  • for the same reason, all objects will be considered equal: uniq([{foo:1},{foo:2}]) will return just [{foo:1}].

That said, if your arrays contain only primitives and you don't care about types (e.g. it's always numbers), this solution is optimal.

The best from two worlds

A universal solution combines both approaches: it uses hash lookups for primitives and linear search for objects.

function uniq(a) {
    var prims = {"boolean":{}, "number":{}, "string":{}}, objs = [];

    return a.filter(function(item) {
        var type = typeof item;
        if(type in prims)
            return prims[type].hasOwnProperty(item) ? false : (prims[type][item] = true);
        else
            return objs.indexOf(item) >= 0 ? false : objs.push(item);
    });
}

sort | uniq

Another option is to sort the array first, and then remove each element equal to the preceding one:

function uniq(a) {
    return a.sort().filter(function(item, pos, ary) {
        return !pos || item != ary[pos - 1];
    });
}

Again, this doesn't work with objects (because all objects are equal for sort). Additionally, we silently change the original array as a side effect - not good! However, if your input is already sorted, this is the way to go (just remove sort from the above).

Unique by...

Sometimes it's desired to uniquify a list based on some criteria other than just equality, for example, to filter out objects that are different, but share some property. This can be done elegantly by passing a callback. This "key" callback is applied to each element, and elements with equal "keys" are removed. Since key is expected to return a primitive, hash table will work fine here:

function uniqBy(a, key) {
    var seen = {};
    return a.filter(function(item) {
        var k = key(item);
        return seen.hasOwnProperty(k) ? false : (seen[k] = true);
    })
}

A particularly useful key() is JSON.stringify which will remove objects that are physically different, but "look" the same:

a = [[1,2,3], [4,5,6], [1,2,3]]
b = uniqBy(a, JSON.stringify)
console.log(b) // [[1,2,3], [4,5,6]]

If the key is not primitive, you have to resort to the linear search:

function uniqBy(a, key) {
    var index = [];
    return a.filter(function (item) {
        var k = key(item);
        return index.indexOf(k) >= 0 ? false : index.push(k);
    });
}

In ES6 you can use a Set:

function uniqBy(a, key) {
    let seen = new Set();
    return a.filter(item => {
        let k = key(item);
        return seen.has(k) ? false : seen.add(k);
    });
}

or a Map:

function uniqBy(a, key) {
    return [
        ...new Map(
            a.map(x => [key(x), x])
        ).values()
    ]
}

which both also work with non-primitive keys.

First or last?

When removing objects by a key, you might to want to keep the first of "equal" objects or the last one.

Use the Set variant above to keep the first, and the Map to keep the last:

function uniqByKeepFirst(a, key) {
    let seen = new Set();
    return a.filter(item => {
        let k = key(item);
        return seen.has(k) ? false : seen.add(k);
    });
}


function uniqByKeepLast(a, key) {
    return [
        ...new Map(
            a.map(x => [key(x), x])
        ).values()
    ]
}

//

data = [
    {a:1, u:1},
    {a:2, u:2},
    {a:3, u:3},
    {a:4, u:1},
    {a:5, u:2},
    {a:6, u:3},
];

console.log(uniqByKeepFirst(data, it => it.u))
console.log(uniqByKeepLast(data, it => it.u))

Libraries

Both underscore and Lo-Dash provide uniq methods. Their algorithms are basically similar to the first snippet above and boil down to this:

var result = [];
a.forEach(function(item) {
     if(result.indexOf(item) < 0) {
         result.push(item);
     }
});

This is quadratic, but there are nice additional goodies, like wrapping native indexOf, ability to uniqify by a key (iteratee in their parlance), and optimizations for already sorted arrays.

If you're using jQuery and can't stand anything without a dollar before it, it goes like this:

  $.uniqArray = function(a) {
        return $.grep(a, function(item, pos) {
            return $.inArray(item, a) === pos;
        });
  }

which is, again, a variation of the first snippet.

Performance

Function calls are expensive in JavaScript, therefore the above solutions, as concise as they are, are not particularly efficient. For maximal performance, replace filter with a loop and get rid of other function calls:

function uniq_fast(a) {
    var seen = {};
    var out = [];
    var len = a.length;
    var j = 0;
    for(var i = 0; i < len; i++) {
         var item = a[i];
         if(seen[item] !== 1) {
               seen[item] = 1;
               out[j++] = item;
         }
    }
    return out;
}

This chunk of ugly code does the same as the snippet #3 above, but an order of magnitude faster (as of 2017 it's only twice as fast - JS core folks are doing a great job!)

function uniq(a) {
    var seen = {};
    return a.filter(function(item) {
        return seen.hasOwnProperty(item) ? false : (seen[item] = true);
    });
}

function uniq_fast(a) {
    var seen = {};
    var out = [];
    var len = a.length;
    var j = 0;
    for(var i = 0; i < len; i++) {
         var item = a[i];
         if(seen[item] !== 1) {
               seen[item] = 1;
               out[j++] = item;
         }
    }
    return out;
}

/////

var r = [0,1,2,3,4,5,6,7,8,9],
    a = [],
    LEN = 1000,
    LOOPS = 1000;

while(LEN--)
    a = a.concat(r);

var d = new Date();
for(var i = 0; i < LOOPS; i++)
    uniq(a);
document.write('<br>uniq, ms/loop: ' + (new Date() - d)/LOOPS)

var d = new Date();
for(var i = 0; i < LOOPS; i++)
    uniq_fast(a);
document.write('<br>uniq_fast, ms/loop: ' + (new Date() - d)/LOOPS)

ES6

ES6 provides the Set object, which makes things a whole lot easier:

function uniq(a) {
   return Array.from(new Set(a));
}

or

let uniq = a => [...new Set(a)];

Note that, unlike in python, ES6 sets are iterated in insertion order, so this code preserves the order of the original array.

However, if you need an array with unique elements, why not use sets right from the beginning?

Generators

A "lazy", generator-based version of uniq can be built on the same basis:

  • take the next value from the argument
  • if it's been seen already, skip it
  • otherwise, yield it and add it to the set of already seen values

function* uniqIter(a) {
    let seen = new Set();

    for (let x of a) {
        if (!seen.has(x)) {
            seen.add(x);
            yield x;
        }
    }
}

// example:

function* randomsBelow(limit) {
    while (1)
        yield Math.floor(Math.random() * limit);
}

// note that randomsBelow is endless

count = 20;
limit = 30;

for (let r of uniqIter(randomsBelow(limit))) {
    console.log(r);
    if (--count === 0)
        break
}

// exercise for the reader: what happens if we set `limit` less than `count` and why

Answered   2023-09-20 20:33:21

  • filter and indexOf have been introduced in ECMAScript 5, so this will not work in old IE versions (<9). If you care about those browsers, you will have to use libraries with similar functions (jQuery, underscore.js etc.) - anyone
  • @RoderickObrist you might if you want your page to work in older browsers - anyone
  • This is O(n^2) solution, which can run very slow in large arrays... - anyone
  • Try this array: ["toString", "valueOf", "failed"]. toString and valueOf are stripped completely. Use Object.create(null) instead of {}. - anyone
  • Anyone know how fast the Set conversion solution is, compared to the others? - anyone

Quick and dirty using jQuery:

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];
var uniqueNames = [];
$.each(names, function(i, el){
    if($.inArray(el, uniqueNames) === -1) uniqueNames.push(el);
});

Answered   2023-09-20 20:33:21

  • As this was reverted back to the original inArray solution by a reputable person, I am going to again mention: this solution is O(n^2), making it inefficient. - anyone
  • I really wish in 2020 we could start depreciating jQuery and other even-more dated answers... Stackoverflow is starting to show some age here... - anyone
  • I agree @NickSteele but I find it does happen naturally over time if you look at votes and not the accepted answer. The best answer will gravitate towards the top as older deprecated answers get downvoted - anyone
  • let uniqueNames = names.filter((item, pos ,self) => self.indexOf(item) == pos); - anyone
  • If you're using jquery, there is $.unique, though that will also sort the items in the result. The best answer is below (creating a set from the array), this answer is inefficient, and out-of-date. - anyone

Got tired of seeing all bad examples with for-loops or jQuery. Javascript has the perfect tools for this nowadays: sort, map and reduce.

Uniq reduce while keeping existing order

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

var uniq = names.reduce(function(a,b){
    if (a.indexOf(b) < 0 ) a.push(b);
    return a;
  },[]);

console.log(uniq, names) // [ 'Mike', 'Matt', 'Nancy', 'Adam', 'Jenny', 'Carl' ]

// one liner
return names.reduce(function(a,b){if(a.indexOf(b)<0)a.push(b);return a;},[]);

Faster uniq with sorting

There are probably faster ways but this one is pretty decent.

var uniq = names.slice() // slice makes copy of array before sorting it
  .sort(function(a,b){
    return a > b;
  })
  .reduce(function(a,b){
    if (a.slice(-1)[0] !== b) a.push(b); // slice(-1)[0] means last item in array without removing it (like .pop())
    return a;
  },[]); // this empty array becomes the starting value for a

// one liner
return names.slice().sort(function(a,b){return a > b}).reduce(function(a,b){if (a.slice(-1)[0] !== b) a.push(b);return a;},[]);

Update 2015: ES6 version:

In ES6 you have Sets and Spread which makes it very easy and performant to remove all duplicates:

var uniq = [ ...new Set(names) ]; // [ 'Mike', 'Matt', 'Nancy', 'Adam', 'Jenny', 'Carl' ]

Sort based on occurrence:

Someone asked about ordering the results based on how many unique names there are:

var names = ['Mike', 'Matt', 'Nancy', 'Adam', 'Jenny', 'Nancy', 'Carl']

var uniq = names
  .map((name) => {
    return {count: 1, name: name}
  })
  .reduce((a, b) => {
    a[b.name] = (a[b.name] || 0) + b.count
    return a
  }, {})

var sorted = Object.keys(uniq).sort((a, b) => uniq[a] < uniq[b])

console.log(sorted)

Answered   2023-09-20 20:33:21

  • Nice! Would it be possible to sort the array based on the frequency of duplicate objects? So that "Nancy" in the above example is moved to the front (or back) of the modified array? - anyone
  • @ALx - I updated with an example for sorting based on occurrence. - anyone
  • sort() appears to be called incorrectly in your second example: if a is < b then it returns the same value as if a == b, which can lead to unsorted results. Unless you're doing something clever here that I'm missing, should be .sort(function(a,b){ return a > b ? 1 : a < b ? -1 : 0; }) - anyone
  • If the data is just an array of names with no requirement other than to eliminate duplicates, why bother with sort, map, and reduce? Just use a set - job done in O(n) time. -- msdn.microsoft.com/en-us/library/dn251547 - anyone
  • @Dave yes- see my example on [...new Set(names)] above - anyone

Vanilla JS: Remove duplicates using an Object like a Set

You can always try putting it into an object, and then iterating through its keys:

function remove_duplicates(arr) {
    var obj = {};
    var ret_arr = [];
    for (var i = 0; i < arr.length; i++) {
        obj[arr[i]] = true;
    }
    for (var key in obj) {
        ret_arr.push(key);
    }
    return ret_arr;
}

Vanilla JS: Remove duplicates by tracking already seen values (order-safe)

Or, for an order-safe version, use an object to store all previously seen values, and check values against it before before adding to an array.

function remove_duplicates_safe(arr) {
    var seen = {};
    var ret_arr = [];
    for (var i = 0; i < arr.length; i++) {
        if (!(arr[i] in seen)) {
            ret_arr.push(arr[i]);
            seen[arr[i]] = true;
        }
    }
    return ret_arr;

}

ECMAScript 6: Use the new Set data structure (order-safe)

ECMAScript 6 adds the new Set Data-Structure, which lets you store values of any type. Set.values returns elements in insertion order.

function remove_duplicates_es6(arr) {
    let s = new Set(arr);
    let it = s.values();
    return Array.from(it);
}

Example usage:

a = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

b = remove_duplicates(a);
// b:
// ["Adam", "Carl", "Jenny", "Matt", "Mike", "Nancy"]

c = remove_duplicates_safe(a);
// c:
// ["Mike", "Matt", "Nancy", "Adam", "Jenny", "Carl"]

d = remove_duplicates_es6(a);
// d:
// ["Mike", "Matt", "Nancy", "Adam", "Jenny", "Carl"]

Answered   2023-09-20 20:33:21

  • In more recent browsers, you could even do var c = Object.keys(b). It should be noted that this approach will only work for strings, but it's alright, that's what the original question was asking for. - anyone
  • It should also be noted that you may lose the order of the array because objects don't keep their properties in order. - anyone
  • @JuanMendes I have created an order-safe version, which simply copies to the new array if the value has not been seen before. - anyone
  • What is happening on this line obj[arr[i]] = true; ?? - anyone
  • @kittu, that is getting the ith element of the array, and putting it into the object (being used as a set). The key is the element, and the value is true, which is entirely arbitrary, as we only care about the keys of the object. - anyone

A single line version using array .filter and .indexOf function:

arr = arr.filter(function (value, index, array) { 
  return array.indexOf(value) === index;
});

ES6 approach

arr = arr.filter((value, index, array) => 
  array.indexOf(value) === index
)

Answered   2023-09-20 20:33:21

  • care to explain how it eliminates dupes? - anyone
  • @web_dev: it doesn't !! I have corrected a previous edit which broke the code. Hope it makes more sens now. Thanks for asking! - anyone
  • This unfortunately has poor performance if this is a large array -- arr.indexOf is O(n), which makes this algorithm O(n^2) - anyone
  • This solution in fact is extremely slow as @CaseyKuball suggests - see stackoverflow.com/questions/67424599/… - anyone

Use Underscore.js

It's a library with a host of functions for manipulating arrays.

It's the tie to go along with jQuery's tux, and Backbone.js's suspenders.

_.uniq

_.uniq(array, [isSorted], [iterator]) Alias: unique
Produces a duplicate-free version of the array, using === to test object equality. If you know in advance that the array is sorted, passing true for isSorted will run a much faster algorithm. If you want to compute unique items based on a transformation, pass an iterator function.

Example

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

alert(_.uniq(names, false));

Note: Lo-Dash (an underscore competitor) also offers a comparable .uniq implementation.

Answered   2023-09-20 20:33:21

  • unfortunately underscore does not provide the ability to define a custom equality function. The callback they do allow is for an 'iteratee' function e.g. with args (item, value, array). - anyone
  • [...new Set(Array)] is more than enough mate - anyone
  • @norbekoff - absolutely, lol. ~10 years later! - anyone

One line:

let names = ['Mike','Matt','Nancy','Adam','Jenny','Nancy','Carl', 'Nancy'];
let dup = [...new Set(names)];
console.log(dup);

Answered   2023-09-20 20:33:21

  • Best answer, if you're using ES6 - anyone
  • what means this 3 dots? - anyone
  • @Vitalicus, that's the spread operator in ES6. Read more here - anyone

You can simply do it in JavaScript, with the help of the second - index - parameter of the filter method:

var a = [2,3,4,5,5,4];
a.filter(function(value, index){ return a.indexOf(value) == index });

or in short hand

a.filter((v,i) => a.indexOf(v) == i)

Answered   2023-09-20 20:33:21

  • this only works for an array containing primitives? - anyone
  • this a.indexOf(v)==i should be a.indexOf(v) === a.lastIndexOf(v) - anyone
  • @Hitmands You are comparing from right , I am comparing from left . nothing else . - anyone
  • Works also without requiring the a variable, as the array is the 3rd parameter of filter: [1/0, 2,1/0,2,3].filter((v,i,a) => a.indexOf(v) === i) (note that it also works nice with Infinity ☺ ) - anyone
  • You can also do .filter((v,i, array) => array.indexOf(v) == i) if you are using this after a map, reduce etc. - anyone

use Array.filter() like this

var actualArr = ['Apple', 'Apple', 'Banana', 'Mango', 'Strawberry', 'Banana'];

console.log('Actual Array: ' + actualArr);

var filteredArr = actualArr.filter(function(item, index) {
  if (actualArr.indexOf(item) == index)
    return item;
});

console.log('Filtered Array: ' + filteredArr);

this can be made shorter in ES6 to

actualArr.filter((item,index,self) => self.indexOf(item)==index);

Here is nice explanation of Array.filter()

Answered   2023-09-20 20:33:21

  • Can you elaborate what you've done here? :-) - anyone
  • doesn't work when the array is an array of arrays - anyone

The most concise way to remove duplicates from an array using native javascript functions is to use a sequence like below:

vals.sort().reduce(function(a, b){ if (b != a[0]) a.unshift(b); return a }, [])

there's no need for slice nor indexOf within the reduce function, like i've seen in other examples! it makes sense to use it along with a filter function though:

vals.filter(function(v, i, a){ return i == a.indexOf(v) })

Yet another ES6(2015) way of doing this that already works on a few browsers is:

Array.from(new Set(vals))

or even using the spread operator:

[...new Set(vals)]

cheers!

Answered   2023-09-20 20:33:21

  • Set is great and very intuitive for those used to python. Too bad they do not have those great (union, intersect, difference) methods. - anyone
  • I went with the simplistic one line of code that utilizes the set mechanic. This was for a custom automation task so I was not leery of using it in the latest version of Chrome (within jsfiddle). However, I would still like to know the shortest all browser compliant way to de-dupe an array. - anyone
  • sets are part of the new specification, you should use the sort/reduce combo to assure cross-browser compatibility @AlexanderDixon - anyone
  • .reduce() is not cross-browser compatible as I would have to apply a poly-fill. I appreciate your response though. developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/… - anyone

The top answers have complexity of O(n²), but this can be done with just O(n) by using an object as a hash:

function getDistinctArray(arr) {
    var dups = {};
    return arr.filter(function(el) {
        var hash = el.valueOf();
        var isDup = dups[hash];
        dups[hash] = true;
        return !isDup;
    });
}

This will work for strings, numbers, and dates. If your array contains objects, the above solution won't work because when coerced to a string, they will all have a value of "[object Object]" (or something similar) and that isn't suitable as a lookup value. You can get an O(n) implementation for objects by setting a flag on the object itself:

function getDistinctObjArray(arr) {
    var distinctArr = arr.filter(function(el) {
        var isDup = el.inArray;
        el.inArray = true;
        return !isDup;
    });
    distinctArr.forEach(function(el) {
        delete el.inArray;
    });
    return distinctArr;
}

2019 edit: Modern versions of JavaScript make this a much easier problem to solve. Using Set will work, regardless of whether your array contains objects, strings, numbers, or any other type.

function getDistinctArray(arr) {
    return [...new Set(arr)];
}

The implementation is so simple, defining a function is no longer warranted.

Answered   2023-09-20 20:33:21

  • Did you consider the performance hit in your method? - anyone
  • @Tushar - Your gist gives a 404. No sorting algorithm has O(n) complexity. Sorting would not be faster. - anyone
  • @Tushar - there are no actual duplicates in that array. If you want to remove objects from an array that have exactly the same properties and values as other objects in the array, you would need to write a custom equality checking function to support it. - anyone
  • @Tushar - None of the answers on this page would remove any duplicates from such an array as is in your gist. - anyone
  • just note that IE is late to the party for Set - anyone

Simplest One I've run into so far. In es6.

 var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl", "Mike", "Nancy"]

 var noDupe = Array.from(new Set(names))

https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set

Answered   2023-09-20 20:33:21

  • For Mac users, even though this is an ES6 function, it works in macOS 10.11.6 El Capitan, using the Script Editor. - anyone

In ECMAScript 6 (aka ECMAScript 2015), Set can be used to filter out duplicates. Then it can be converted back to an array using the spread operator.

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"],
    unique = [...new Set(names)];

Answered   2023-09-20 20:33:21

  • the constructor of Set actually requires the new keyword - anyone
  • @Ivo Thanks. Previously Firefox's implementation didn't require new, I wonder if the ES6 draft changed about this behavior. - anyone
  • some constructors might be indeed called as functions though this kind of behaviour depends on the browser's implementation of the spec ;) - anyone

Solution 1

Array.prototype.unique = function() {
    var a = [];
    for (i = 0; i < this.length; i++) {
        var current = this[i];
        if (a.indexOf(current) < 0) a.push(current);
    }
    return a;
}

Solution 2 (using Set)

Array.prototype.unique = function() {
    return Array.from(new Set(this));
}

Test

var x=[1,2,3,3,2,1];
x.unique() //[1,2,3]

Performance

When I tested both implementation (with and without Set) for performance in chrome, I found that the one with Set is much much faster!

Array.prototype.unique1 = function() {
    var a = [];
    for (i = 0; i < this.length; i++) {
        var current = this[i];
        if (a.indexOf(current) < 0) a.push(current);
    }
    return a;
}


Array.prototype.unique2 = function() {
    return Array.from(new Set(this));
}

var x=[];
for(var i=0;i<10000;i++){
	x.push("x"+i);x.push("x"+(i+1));
}

console.time("unique1");
console.log(x.unique1());
console.timeEnd("unique1");



console.time("unique2");
console.log(x.unique2());
console.timeEnd("unique2");

Answered   2023-09-20 20:33:21

  • Upvote for the use of Set. I don't know the performance comparison though - anyone
  • I have read somewhere that an Array is faster than a Set (overall performance), But when I tested in chrome, the implementation with Set was much much faster! see the edited answer :) - anyone
  • better practice is to use Object.defineProperty(Array.prototype,"unique".. instead of Array.prototype.unique = ... See more info here stackoverflow.com/questions/10105824/… - anyone
  • the Set approach doesn't seem to work for me in Node. new Set([5,5]) seems to return [5,5] in some cases. I'm as baffled as you are. Edit: I found out what's happening. new Set([new Number(5), new Number(5)]) returns [5,5]. Apparently Node thinks the two number 5s are different if I instantiate them with new... which is honestly the stupidest thing I've ever seen. - anyone
  • @Demonblack This is a valid concern. x=new Number(5) and another y=new Number(5) will be two different Objects, as oppose to just var x=5 and var y=5. new keyword will create a new object. I know this explanation is obvious but that's all I know :) - anyone

Go for this one:

var uniqueArray = duplicateArray.filter(function(elem, pos) {
    return duplicateArray.indexOf(elem) == pos;
}); 

Now uniqueArray contains no duplicates.

Answered   2023-09-20 20:33:21

The following is more than 80% faster than the jQuery method listed (see tests below). It is an answer from a similar question a few years ago. If I come across the person who originally proposed it I will post credit. Pure JS.

var temp = {};
for (var i = 0; i < array.length; i++)
  temp[array[i]] = true;
var r = [];
for (var k in temp)
  r.push(k);
return r;

My test case comparison: http://jsperf.com/remove-duplicate-array-tests

Answered   2023-09-20 20:33:21

  • I add a more fast version in revision 4. Please, review! - anyone
  • the test didn't seem to be using arrays??? i've added (yet another) one that seems to be consistently fast over different browsers (see jsperf.com/remove-duplicate-array-tests/10) : for (var n = array.length, result = [array[n--]], i; n--;) { i = array[n]; if (!(i in result)) result.push(i); } return result; - anyone

I had done a detailed comparison of dupes removal at some other question but having noticed that this is the real place i just wanted to share it here as well.

I believe this is the best way to do this

var myArray = [100, 200, 100, 200, 100, 100, 200, 200, 200, 200],
    reduced = Object.keys(myArray.reduce((p,c) => (p[c] = true,p),{}));
console.log(reduced);

OK .. even though this one is O(n) and the others are O(n^2) i was curious to see benchmark comparison between this reduce / look up table and filter/indexOf combo (I choose Jeetendras very nice implementation https://stackoverflow.com/a/37441144/4543207). I prepare a 100K item array filled with random positive integers in range 0-9999 and and it removes the duplicates. I repeat the test for 10 times and the average of the results show that they are no match in performance.

  • In firefox v47 reduce & lut : 14.85ms vs filter & indexOf : 2836ms
  • In chrome v51 reduce & lut : 23.90ms vs filter & indexOf : 1066ms

Well ok so far so good. But let's do it properly this time in the ES6 style. It looks so cool..! But as of now how it will perform against the powerful lut solution is a mystery to me. Lets first see the code and then benchmark it.

var myArray = [100, 200, 100, 200, 100, 100, 200, 200, 200, 200],
    reduced = [...myArray.reduce((p,c) => p.set(c,true),new Map()).keys()];
console.log(reduced);

Wow that was short..! But how about the performance..? It's beautiful... Since the heavy weight of the filter / indexOf lifted over our shoulders now i can test an array 1M random items of positive integers in range 0..99999 to get an average from 10 consecutive tests. I can say this time it's a real match. See the result for yourself :)

var ranar = [],
     red1 = a => Object.keys(a.reduce((p,c) => (p[c] = true,p),{})),
     red2 = a => reduced = [...a.reduce((p,c) => p.set(c,true),new Map()).keys()],
     avg1 = [],
     avg2 = [],
       ts = 0,
       te = 0,
     res1 = [],
     res2 = [],
     count= 10;
for (var i = 0; i<count; i++){
  ranar = (new Array(1000000).fill(true)).map(e => Math.floor(Math.random()*100000));
  ts = performance.now();
  res1 = red1(ranar);
  te = performance.now();
  avg1.push(te-ts);
  ts = performance.now();
  res2 = red2(ranar);
  te = performance.now();
  avg2.push(te-ts);
}

avg1 = avg1.reduce((p,c) => p+c)/count;
avg2 = avg2.reduce((p,c) => p+c)/count;

console.log("reduce & lut took: " + avg1 + "msec");
console.log("map & spread took: " + avg2 + "msec");

Which one would you use..? Well not so fast...! Don't be deceived. Map is at displacement. Now look... in all of the above cases we fill an array of size n with numbers of range < n. I mean we have an array of size 100 and we fill with random numbers 0..9 so there are definite duplicates and "almost" definitely each number has a duplicate. How about if we fill the array in size 100 with random numbers 0..9999. Let's now see Map playing at home. This time an Array of 100K items but random number range is 0..100M. We will do 100 consecutive tests to average the results. OK let's see the bets..! <- no typo

var ranar = [],
     red1 = a => Object.keys(a.reduce((p,c) => (p[c] = true,p),{})),
     red2 = a => reduced = [...a.reduce((p,c) => p.set(c,true),new Map()).keys()],
     avg1 = [],
     avg2 = [],
       ts = 0,
       te = 0,
     res1 = [],
     res2 = [],
     count= 100;
for (var i = 0; i<count; i++){
  ranar = (new Array(100000).fill(true)).map(e => Math.floor(Math.random()*100000000));
  ts = performance.now();
  res1 = red1(ranar);
  te = performance.now();
  avg1.push(te-ts);
  ts = performance.now();
  res2 = red2(ranar);
  te = performance.now();
  avg2.push(te-ts);
}

avg1 = avg1.reduce((p,c) => p+c)/count;
avg2 = avg2.reduce((p,c) => p+c)/count;

console.log("reduce & lut took: " + avg1 + "msec");
console.log("map & spread took: " + avg2 + "msec");

Now this is the spectacular comeback of Map()..! May be now you can make a better decision when you want to remove the dupes.

Well ok we are all happy now. But the lead role always comes last with some applause. I am sure some of you wonder what Set object would do. Now that since we are open to ES6 and we know Map is the winner of the previous games let us compare Map with Set as a final. A typical Real Madrid vs Barcelona game this time... or is it? Let's see who will win the el classico :)

var ranar = [],
     red1 = a => reduced = [...a.reduce((p,c) => p.set(c,true),new Map()).keys()],
     red2 = a => Array.from(new Set(a)),
     avg1 = [],
     avg2 = [],
       ts = 0,
       te = 0,
     res1 = [],
     res2 = [],
     count= 100;
for (var i = 0; i<count; i++){
  ranar = (new Array(100000).fill(true)).map(e => Math.floor(Math.random()*10000000));
  ts = performance.now();
  res1 = red1(ranar);
  te = performance.now();
  avg1.push(te-ts);
  ts = performance.now();
  res2 = red2(ranar);
  te = performance.now();
  avg2.push(te-ts);
}

avg1 = avg1.reduce((p,c) => p+c)/count;
avg2 = avg2.reduce((p,c) => p+c)/count;

console.log("map & spread took: " + avg1 + "msec");
console.log("set & A.from took: " + avg2 + "msec");

Wow.. man..! Well unexpectedly it didn't turn out to be an el classico at all. More like Barcelona FC against CA Osasuna :))

Answered   2023-09-20 20:33:21

  • Just btw I get arr.reduce(...).keys(...).slice is not a function in Typescript trying to use your ES6 method - anyone

Here is a simple answer to the question.

var names = ["Alex","Tony","James","Suzane", "Marie", "Laurence", "Alex", "Suzane", "Marie", "Marie", "James", "Tony", "Alex"];
var uniqueNames = [];

    for(var i in names){
        if(uniqueNames.indexOf(names[i]) === -1){
            uniqueNames.push(names[i]);
        }
    }

Answered   2023-09-20 20:33:21

  • +1 for === . It wont work for arrays with mixed types if we dont check for it types. Simple but effective answer - anyone
  • should be the main answer ! - anyone

A simple but effective technique, is to use the filter method in combination with the filter function(value, index){ return this.indexOf(value) == index }.

Code example :

var data = [2,3,4,5,5,4];
var filter = function(value, index){ return this.indexOf(value) == index };
var filteredData = data.filter(filter, data );

document.body.innerHTML = '<pre>' + JSON.stringify(filteredData, null, '\t') +  '</pre>';

See also this Fiddle.

Answered   2023-09-20 20:33:21

  • Genious! And, for instances, if you want to have the repeated ones, (instead of removing them) all you have to do is replace this.indexOf(value) == index by this.indexOf(value, index+1) > 0 Thanks! - anyone
  • You could even resume it to a single "filter" line: filterData = data.filter((v, i) => (data.indexOf(v) == i) ); - anyone
  • Last time I bother! Sorry... picking up my 1st answer, in 2 lines you could get a JSON var JSON_dupCounter = {}; with the repeated ones and how many times they were repeated: data.filter((testItem, index) => (data.indexOf(testItem, index + 1) > 0)).forEach((found_duplicated) => (JSON_dupCounter[found_duplicated] = (JSON_dupCounter [found_duplicated] || 1) + 1)); - anyone
  • this only works for arrays of primitives? - anyone
  • @frozen : If works with everything where == can be used to determine equality. So, if you're dealing with eg. arrays, objects or functions, the filter will work only for different entries that are references to the same array, object or function (see demo). If you want to determine equality based on different criteria, you'll need to include those criteria in your filter. - anyone

So the options is:

let a = [11,22,11,22];
let b = []


b = [ ...new Set(a) ];     
// b = [11, 22]

b = Array.from( new Set(a))   
// b = [11, 22]

b = a.filter((val,i)=>{
  return a.indexOf(val)==i
})                        
// b = [11, 22]

Answered   2023-09-20 20:33:21

Here is very simple for understanding and working anywhere (even in PhotoshopScript) code. Check it!

var peoplenames = new Array("Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl");

peoplenames = unique(peoplenames);
alert(peoplenames);

function unique(array){
    var len = array.length;
    for(var i = 0; i < len; i++) for(var j = i + 1; j < len; j++) 
        if(array[j] == array[i]){
            array.splice(j,1);
            j--;
            len--;
        }
    return array;
}

//*result* peoplenames == ["Mike","Matt","Nancy","Adam","Jenny","Carl"]

Answered   2023-09-20 20:33:21

here is the simple method without any special libraries are special function,

name_list = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];
get_uniq = name_list.filter(function(val,ind) { return name_list.indexOf(val) == ind; })

console.log("Original name list:"+name_list.length, name_list)
console.log("\n Unique name list:"+get_uniq.length, get_uniq)

enter image description here

Answered   2023-09-20 20:33:21

Apart from being a simpler, more terse solution than the current answers (minus the future-looking ES6 ones), I perf tested this and it was much faster as well:

var uniqueArray = dupeArray.filter(function(item, i, self){
  return self.lastIndexOf(item) == i;
});

One caveat: Array.lastIndexOf() was added in IE9, so if you need to go lower than that, you'll need to look elsewhere.

Answered   2023-09-20 20:33:21

Generic Functional Approach

Here is a generic and strictly functional approach with ES2015:

// small, reusable auxiliary functions

const apply = f => a => f(a);

const flip = f => b => a => f(a) (b);

const uncurry = f => (a, b) => f(a) (b);

const push = x => xs => (xs.push(x), xs);

const foldl = f => acc => xs => xs.reduce(uncurry(f), acc);

const some = f => xs => xs.some(apply(f));


// the actual de-duplicate function

const uniqueBy = f => foldl(
   acc => x => some(f(x)) (acc)
    ? acc
    : push(x) (acc)
 ) ([]);


// comparators

const eq = y => x => x === y;

// string equality case insensitive :D
const seqCI = y => x => x.toLowerCase() === y.toLowerCase();


// mock data

const xs = [1,2,3,1,2,3,4];

const ys = ["a", "b", "c", "A", "B", "C", "D"];


console.log( uniqueBy(eq) (xs) );

console.log( uniqueBy(seqCI) (ys) );

We can easily derive unique from unqiueBy or use the faster implementation utilizing Sets:

const unqiue = uniqueBy(eq);

// const unique = xs => Array.from(new Set(xs));

Benefits of this approach:

  • generic solution by using a separate comparator function
  • declarative and succinct implementation
  • reuse of other small, generic functions

Performance Considerations

uniqueBy isn't as fast as an imperative implementation with loops, but it is way more expressive due to its genericity.

If you identify uniqueBy as the cause of a concrete performance penalty in your app, replace it with optimized code. That is, write your code first in an functional, declarative way. Afterwards, provided that you encounter performance issues, try to optimize the code at the locations, which are the cause of the problem.

Memory Consumption and Garbage Collection

uniqueBy utilizes mutations (push(x) (acc)) hidden inside its body. It reuses the accumulator instead of throwing it away after each iteration. This reduces memory consumption and GC pressure. Since this side effect is wrapped inside the function, everything outside remains pure.

Answered   2023-09-20 20:33:21

for (i=0; i<originalArray.length; i++) {  
    if (!newArray.includes(originalArray[i])) {
        newArray.push(originalArray[i]); 
    }
}

Answered   2023-09-20 20:33:21

The following script returns a new array containing only unique values. It works on string and numbers. No requirement for additional libraries only vanilla JS.

Browser support:

Feature Chrome  Firefox (Gecko)     Internet Explorer   Opera   Safari
Basic support   (Yes)   1.5 (1.8)   9                   (Yes)   (Yes)

https://jsfiddle.net/fzmcgcxv/3/

var duplicates = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl","Mike","Mike","Nancy","Carl"]; 
var unique = duplicates.filter(function(elem, pos) {
    return duplicates.indexOf(elem) == pos;
  }); 
alert(unique);

Answered   2023-09-20 20:33:21

If by any chance you were using

D3.js

You could do

d3.set(["foo", "bar", "foo", "baz"]).values() ==> ["foo", "bar", "baz"]

https://github.com/mbostock/d3/wiki/Arrays#set_values

Answered   2023-09-20 20:33:21

  • Beautiful, but loading the full fledged powerful visualization library to only filter duplicates seems overkill. Lucky I need the lib for some purpose and I will be using this. Thanks very much. - anyone

A slight modification of thg435's excellent answer to use a custom comparator:

function contains(array, obj) {
    for (var i = 0; i < array.length; i++) {
        if (isEqual(array[i], obj)) return true;
    }
    return false;
}
//comparator
function isEqual(obj1, obj2) {
    if (obj1.name == obj2.name) return true;
    return false;
}
function removeDuplicates(ary) {
    var arr = [];
    return ary.filter(function(x) {
        return !contains(arr, x) && arr.push(x);
    });
}

Answered   2023-09-20 20:33:21

$(document).ready(function() {

    var arr1=["dog","dog","fish","cat","cat","fish","apple","orange"]

    var arr2=["cat","fish","mango","apple"]

    var uniquevalue=[];
    var seconduniquevalue=[];
    var finalarray=[];

    $.each(arr1,function(key,value){

       if($.inArray (value,uniquevalue) === -1)
       {
           uniquevalue.push(value)

       }

    });

     $.each(arr2,function(key,value){

       if($.inArray (value,seconduniquevalue) === -1)
       {
           seconduniquevalue.push(value)

       }

    });

    $.each(uniquevalue,function(ikey,ivalue){

        $.each(seconduniquevalue,function(ukey,uvalue){

            if( ivalue == uvalue)

            {
                finalarray.push(ivalue);
            }   

        });

    });
    alert(finalarray);
});

Answered   2023-09-20 20:33:21

https://jsfiddle.net/2w0k5tz8/

function remove_duplicates(array_){
    var ret_array = new Array();
    for (var a = array_.length - 1; a >= 0; a--) {
        for (var b = array_.length - 1; b >= 0; b--) {
            if(array_[a] == array_[b] && a != b){
                delete array_[b];
            }
        };
        if(array_[a] != undefined)
            ret_array.push(array_[a]);
    };
    return ret_array;
}

console.log(remove_duplicates(Array(1,1,1,2,2,2,3,3,3)));

Loop through, remove duplicates, and create a clone array place holder because the array index will not be updated.

Loop backward for better performance ( your loop wont need to keep checking the length of your array)

Answered   2023-09-20 20:33:21